[ Mar 30 04:02:07 Enabled. ]
[ Mar 30 04:02:07 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ Mar 30 04:02:07 Method "start" exited with status 0. ]
[ Mar 30 04:02:07 Rereading configuration. ]
[ Mar 30 04:02:07 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.120686743Z","hostname":"sock","pid":1618,"file":"illumos-utils/src/running_zone.rs:163"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.120835617Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:253"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.173424061Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:260","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.173474062Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.290878844Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.299457734Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.307706116Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:270","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:07.911898087Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.057348922Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:115"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.057385413Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:127"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.057494505Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:137"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.057656199Z","hostname":"sock","pid":1618,"component":"DumpSetup-archival","file":"sled-agent/src/dump_setup.rs:291"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.05766974Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:157","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.05768117Z","hostname":"sock","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.057728601Z","hostname":"sock","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:609"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060787492Z","hostname":"sock","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:08.060800802Z","hostname":"sock","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:658"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060807693Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:171"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060815753Z","hostname":"sock","pid":1618,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060842073Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060883734Z","hostname":"sock","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:261","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060948136Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060958306Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.060977286Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.07357357Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.0736102Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:312","id":"4d567028-d7e0-4e3c-9e3e-6a96da036239"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.174438367Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.379897969Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.379929649Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.37993522Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.37994822Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.37995236Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.37995695Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.388757745Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.388789866Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:312","id":"c653edc4-5059-41c6-afb0-ff8ae7c3fb4a"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.478453372Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685279496Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_0.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685326277Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685365308Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685373568Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685387398Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685428889Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.68544644Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.68546378Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685499101Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685510281Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685514731Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685543252Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685548922Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685553142Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685567533Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685575303Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685580483Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685597923Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685604623Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685609434Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685627834Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685643104Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685647434Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685663235Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685678875Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685686845Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685690915Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685695805Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685718316Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685723746Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685727926Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685742497Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685764097Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:227","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685769167Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685775047Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685780518Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685810668Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:87"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685822788Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:89"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685826988Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685831249Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685856209Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:195"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.68590563Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685922561Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685932891Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685948621Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685956331Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685961282Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685991482Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:200"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.685996372Z","hostname":"sock","pid":1618,"file":"sled-agent/src/long_running_tasks.rs:215"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.696710032Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:701"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.696771383Z","hostname":"sock","pid":1618,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999926278s","next_cleanup":"Instant { tv_sec: 2731, tv_nsec: 909593642 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.696782823Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3588"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.696819404Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.696877296Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.699756753Z","hostname":"sock","pid":1618,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700142562Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700232574Z","hostname":"sock","pid":1618,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.2.164,\n                last: 10.151.2.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.2.164,\n        10.151.2.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV1 {\n        rack_subnet: Ipv6Network {\n            addr: fd00:1122:3344:100::,\n            prefix: 56,\n        },\n        infra_ip_first: 10.151.2.174,\n        infra_ip_last: 10.151.2.174,\n        ports: [\n            PortConfigV1 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Network {\n                                addr: 0.0.0.0,\n                                prefix: 0,\n                            },\n                        ),\n                        nexthop: 10.151.2.100,\n                    },\n                ],\n                addresses: [\n                    V4(\n                        Ipv4Network {\n                            addr: 10.151.2.174,\n                            prefix: 24,\n                        },\n                    ),\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.70049959Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:997"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.70050648Z","hostname":"sock","pid":1618,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700557501Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700601972Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700618763Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700624533Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1057"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700655824Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700668284Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700687664Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700693645Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1098"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700698065Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700702355Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700730035Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.700757956Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701139705Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701309829Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701318499Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1131"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.7013577Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.70136319Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701513664Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701631866Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701687378Z","hostname":"sock","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 82d35b43-2f1a-463b-bc45-db2841fe70e1, rack_id: fffdd475-94d1-496c-86ec-4c108167d272, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701839571Z","hostname":"sock","pid":1618,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55148","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 82d35b43-2f1a-463b-bc45-db2841fe70e1, rack_id: fffdd475-94d1-496c-86ec-4c108167d272, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701969954Z","hostname":"sock","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:356"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.701976834Z","hostname":"sock","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:371"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702001665Z","hostname":"sock","pid":1618,"file":"sled-agent/src/server.rs:44"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702006795Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702010695Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702014655Z","hostname":"sock","pid":1618,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702038786Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702045246Z","hostname":"sock","pid":1618,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702050906Z","hostname":"sock","pid":1618,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702055456Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.702063066Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.704919303Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3819"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.712663343Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.722824209Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.733219562Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.743400059Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.752238944Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.760534607Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.7688216Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.777101423Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.785417156Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.785474728Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.785482778Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.785487828Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.785495388Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.785502678Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.805280419Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.822046459Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.836888434Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.836955756Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.836970456Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.836995157Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 96.492047ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.840625791Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5a1e78f2-a31e-4de4-9b5a-a0ce0d3cfce9\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5a1e78f2-a31e-4de4-9b5a-a0ce0d3cfce9\" }"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_4d567028-d7e0-4e3c-9e3e-6a96da036239/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.850198484Z","hostname":"sock","pid":1618,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"successfully created encrypted zvol: name=\"oxi_4d567028-d7e0-4e3c-9e3e-6a96da036239/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.874314715Z","hostname":"sock","pid":1618,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_4d567028-d7e0-4e3c-9e3e-6a96da036239/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.874353926Z","hostname":"sock","pid":1618,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.879193249Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.879211819Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"}
{"msg":"Ensuring dataset oxi_4d567028-d7e0-4e3c-9e3e-6a96da036239/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.879217979Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.93727688Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 117.204257ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.937832984Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d388f61e-2b48-40a8-9316-a5776f242459\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d388f61e-2b48-40a8-9316-a5776f242459\" }"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:08.94755841Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.055673045Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 187.108058ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.056286759Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"34257a0c-7157-4076-a99f-26a169ecc0da\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"34257a0c-7157-4076-a99f-26a169ecc0da\" }"}
{"msg":"Mounting oxi_4d567028-d7e0-4e3c-9e3e-6a96da036239/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.165731276Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.177639193Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.177730536Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.177767107Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.177784177Z","hostname":"sock","pid":1618,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.244973301Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 201.170617ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.245463032Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"31ddf7b8-5bbb-4599-b8eb-24c54d22e642\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"31ddf7b8-5bbb-4599-b8eb-24c54d22e642\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.447479684Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 127.489092ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.448318093Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"54e2dac4-b712-41ce-b0c0-93ec8d20b73c\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"54e2dac4-b712-41ce-b0c0-93ec8d20b73c\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.576762642Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 140.076316ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.577335576Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7a4207d9-324d-4c69-8add-bb0e2e5f4e3c\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7a4207d9-324d-4c69-8add-bb0e2e5f4e3c\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.718414899Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 310.879137ms","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.719102795Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"359c7cea-3a3b-4131-8a04-0aab7e853bcb\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"359c7cea-3a3b-4131-8a04-0aab7e853bcb\" }"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.88087691Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.925820746Z","hostname":"sock","pid":1618,"component":"VmmReservoirManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:266"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.925877297Z","hostname":"sock","pid":1618,"component":"VmmReservoirManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:274"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.92599045Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:983","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.92600647Z","hostname":"sock","pid":1618,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927016154Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:195"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927046764Z","hostname":"sock","pid":1618,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:426"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927068145Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927080265Z","hostname":"sock","pid":1618,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927315331Z","hostname":"sock","pid":1618,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:09.927384792Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"315.640844ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927484975Z","hostname":"sock","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927509495Z","hostname":"sock","pid":1618,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:261","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100 }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927518745Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3588"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:09.927523765Z","hostname":"sock","pid":1618,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:55148","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.123494205Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.123966516Z","hostname":"sock","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.123987127Z","hostname":"sock","pid":1618,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.124316035Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.124382476Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.124391366Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.124406307Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.124423077Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:10.129220409Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.156633497Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:65354"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.156811501Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.156910603Z","hostname":"sock","pid":1618,"uri":"/inventory","method":"GET","req_id":"eebce825-6979-417d-8c98-cb8c23a8725d","remote_addr":"[fd00:1122:3344:101::1]:65354","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":165,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.185858147Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:43625"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.185937319Z","hostname":"sock","pid":1618,"uri":"/sled-role","method":"GET","req_id":"6db35af9-dec6-4c14-931c-79e6b86cb39e","remote_addr":"[fd00:1122:3344:101::1]:43625","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":38,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.186588264Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.186600834Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.187000883Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.187232479Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:802"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 52cf60f0-628e-4c34-ab4b-6889b6b84c43, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd }, OmicronPhysicalDiskConfig { id: 8b83e7cb-0f8c-4a09-9620-d3e5e284a365, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 3fdd1e8f-902b-4182-b368-f65989a017ec }, OmicronPhysicalDiskConfig { id: d4c8bd6a-016e-4822-96ee-3ffd92abe8e2, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a }, OmicronPhysicalDiskConfig { id: 3f1cd95c-fd3c-4dd0-bc6e-327b64df8bc6, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: c9f8e23e-40d6-46e7-980e-8af586658d29 }, OmicronPhysicalDiskConfig { id: b352b978-38be-42ba-9077-808cef2d8508, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: ece5c680-a159-4726-9af2-b8b9e799b8a0 }, OmicronPhysicalDiskConfig { id: c2ae46c0-29c7-49ac-af2d-0d6286151028, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc }, OmicronPhysicalDiskConfig { id: 3f7dae4b-8180-4d3c-83cc-94c5af9f96fd, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8 }, OmicronPhysicalDiskConfig { id: e1421a3e-5405-4a3f-9acb-07be4ca0d20c, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 64f2bb49-5808-4ece-aabc-b25bc06f3536 }, OmicronPhysicalDiskConfig { id: 012e73c7-a733-4a8e-8215-158a3458a9f5, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0 }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.218836774Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:344"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.218989038Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:60740"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: 52cf60f0-628e-4c34-ab4b-6889b6b84c43, pool_id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 8b83e7cb-0f8c-4a09-9620-d3e5e284a365, pool_id: 3fdd1e8f-902b-4182-b368-f65989a017ec }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: d4c8bd6a-016e-4822-96ee-3ffd92abe8e2, pool_id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: 3f1cd95c-fd3c-4dd0-bc6e-327b64df8bc6, pool_id: c9f8e23e-40d6-46e7-980e-8af586658d29 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: b352b978-38be-42ba-9077-808cef2d8508, pool_id: ece5c680-a159-4726-9af2-b8b9e799b8a0 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: c2ae46c0-29c7-49ac-af2d-0d6286151028, pool_id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 3f7dae4b-8180-4d3c-83cc-94c5af9f96fd, pool_id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: e1421a3e-5405-4a3f-9acb-07be4ca0d20c, pool_id: 64f2bb49-5808-4ece-aabc-b25bc06f3536 }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: 012e73c7-a733-4a8e-8215-158a3458a9f5, pool_id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0 }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219270925Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219277345Z","hostname":"sock","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219283335Z","hostname":"sock","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219288225Z","hostname":"sock","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219324606Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219330246Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219336246Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.219340456Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:10.304293373Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"294.392869ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.310489727Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:10.310499748Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"3261c18c-ce8f-4256-bb9a-b916c37c65cd"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:10.677890188Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"556.614685ms","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:11.094086633Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:11.264672213Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"1.329686294s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.274044961Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.440301971Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.440410483Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.440422173Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.440440484Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.440453874Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.440608138Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:11.440615068Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:12.007815978Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:12.010346957Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:12.264548883Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Automatically destroying dataset: oxp_3261c18c-ce8f-4256-bb9a-b916c37c65cd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:12.270542203Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:12.611474427Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"4.261187346s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:12.786679605Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 59.222868ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:13.551705918Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:13.552038306Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 67.783616ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:14.204403368Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:14.205494013Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 232.868792ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:14.816163015Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:14.817778232Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_3261c18c-ce8f-4256-bb9a-b916c37c65cd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.012265159Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.478002758Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.478115131Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.478122261Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.478128171Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.478132141Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 546.332287ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:15.696134974Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.702631236Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:15.702644616Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"3fdd1e8f-902b-4182-b368-f65989a017ec"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:15.930593201Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:16.87416467Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 993.781686ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:16.874280472Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:16.874512468Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"4.541351638s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:17.123283138Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.134829146Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.373260195Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.373345657Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.373357838Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.373365688Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.373374928Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.37346649Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:17.37348327Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.241098923s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:18.351514671Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:18.352802291Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:18.353543329Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:18.621422693Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Automatically destroying dataset: oxp_3fdd1e8f-902b-4182-b368-f65989a017ec/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:18.627432853Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:19.424976233Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.365231565Z","hostname":"sock","pid":1618,"component":"VmmReservoirManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:278"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:20.370990499Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.394526607Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2286"}
{"msg":"Looking for unencrypted datasets in oxp_3fdd1e8f-902b-4182-b368-f65989a017ec","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.435940801Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:20.441224914Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.448423311Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.448508704Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.448517384Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.448523654Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.448533664Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.459508789Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.45954278Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"998e9ff7-344d-4c0f-ac09-09fe8899d73a"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.551233914Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.558202326Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.558288538Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.558302169Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.558330759Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.55836058Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.558495733Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.558510384Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.591703636Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_998e9ff7-344d-4c0f-ac09-09fe8899d73a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.598562876Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_998e9ff7-344d-4c0f-ac09-09fe8899d73a","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.715278562Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.722303225Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.722356417Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.722365407Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.722376107Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.722417698Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.732462652Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.732513043Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"c9f8e23e-40d6-46e7-980e-8af586658d29"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.826403268Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.83420377Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.834267091Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.834281102Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.834294792Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.834323012Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.834470416Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.834483066Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.867763351Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_c9f8e23e-40d6-46e7-980e-8af586658d29/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.875373888Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_c9f8e23e-40d6-46e7-980e-8af586658d29","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:20.996101347Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.004689087Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.004738388Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.004747238Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.004753468Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.004758628Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.014568157Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.014602278Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"ece5c680-a159-4726-9af2-b8b9e799b8a0"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.107175272Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.115471125Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.115520646Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.115529126Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.115543307Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.115558967Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.11570846Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.115733001Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.151548534Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_ece5c680-a159-4726-9af2-b8b9e799b8a0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.159034709Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_ece5c680-a159-4726-9af2-b8b9e799b8a0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.283301551Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.290955449Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.29101093Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.291021171Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.291032691Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.291040891Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.300687936Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.300728126Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"c1ff3dbe-98c6-4a12-9904-ae7e718611dc"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.393860134Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.400724893Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.400774675Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.400798635Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.400812236Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.400824856Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.400947089Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.400956909Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"failed to register as a metric producer with Nexus","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:21.418421645Z","hostname":"sock","pid":1618,"sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:1178","retry_after":"11.51146953s","error":"\"Metric registration error: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/metrics/producers): error trying to connect: dns error: proto error: io error: No route to host (os error 148)\""}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.432678997Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_c1ff3dbe-98c6-4a12-9904-ae7e718611dc/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.439459795Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:21.443720414Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_c1ff3dbe-98c6-4a12-9904-ae7e718611dc","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.556336165Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.564377912Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.564438684Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.564446104Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.564452874Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.564458714Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.574244192Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.574276833Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"121748b7-d12c-483c-bb9b-e9f8929f1ec8"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.668464595Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.676685856Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.676731707Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.676739057Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.676745867Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.676756218Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.676937052Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.676959712Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.714220309Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_121748b7-d12c-483c-bb9b-e9f8929f1ec8/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.721748475Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_121748b7-d12c-483c-bb9b-e9f8929f1ec8","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.847625954Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.855533488Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.855587259Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.85559759Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.85560351Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.85561243Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.865649974Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.865682034Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"64f2bb49-5808-4ece-aabc-b25bc06f3536"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.958965105Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.967050623Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.967108844Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.967119754Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.967130495Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.967201656Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.96734778Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:21.96736031Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.00386124Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_64f2bb49-5808-4ece-aabc-b25bc06f3536/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.012642654Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_64f2bb49-5808-4ece-aabc-b25bc06f3536","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.14093794Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.14911826Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.149170881Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.149179842Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.149187992Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.149195032Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/disk.rs:111","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.159371429Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:301"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.159400249Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-hardware/src/disk.rs:307","id":"1eee6f0c-2a27-4133-94bc-978e69e92ff0"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.258057445Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.266198795Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.266251176Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.266264976Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.266278777Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.266327368Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.266507762Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:38","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.266526353Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.303939453Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:57"}
{"msg":"Automatically destroying dataset: oxp_1eee6f0c-2a27-4133-94bc-978e69e92ff0/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.311905249Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:22.368216449Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Looking for unencrypted datasets in oxp_1eee6f0c-2a27-4133-94bc-978e69e92ff0","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.433814286Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.44173577Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.441783361Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.441854253Z","hostname":"sock","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.441992386Z","hostname":"sock","pid":1618,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 4d567028-d7e0-4e3c-9e3e-6a96da036239, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: c653edc4-5059-41c6-afb0-ff8ae7c3fb4a, kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.442274052Z","hostname":"sock","pid":1618,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.442612931Z","hostname":"sock","pid":1618,"uri":"/omicron-physical-disks","method":"PUT","req_id":"7f5ea21f-dbf8-4d32-9dd0-3a8e7afb0af3","remote_addr":"[fd00:1122:3344:101::1]:60740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12223532,"response_code":"200"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:22.445268212Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.474198335Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.474435511Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:39432"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.474676227Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.474681467Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.524357553Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.524392334Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.574255394Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.574285465Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.624086114Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.624112255Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.624127815Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.624164076Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.667073654Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.667118385Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.667122296Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.720317043Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.724315936Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.728580376Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.740710678Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.745318735Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:22.749647216Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:23.4476502Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:24.368558731Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/82d35b43-2f1a-463b-bc45-db2841fe70e1): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-03-30T04:02:24.449660688Z","hostname":"sock","pid":1618,"component":"ProbeManager","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:24.462540158Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:24.462555179Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"}
{"msg":"Profile for oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:3::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:3::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:3::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:3::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:25.078249097Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:25.07833697Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:25.078621146Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:1::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:1::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:1::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:1::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:25.563364767Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:25.563422729Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"}
{"msg":"Profile for oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:2::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:2::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:2::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:2::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:26.044099915Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 60.317562ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.052241944Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:26.052300376Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:26.056724519Z","hostname":"sock","pid":1618,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:26.056810621Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 51.893506ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.123074303Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 53.034967ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.175300378Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 188.464905ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.182457065Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 46.261029ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.189185762Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 130.494732ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.236747828Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 112.659023ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.253421646Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 170.831136ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.442044016Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 416.167692ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.498895759Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 260.988731ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.528724904Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 449.086919ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.662687411Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 332.042336ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.802307711Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 649.54081ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:26.924209908Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 730.439727ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:27.121665983Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 404.084354ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:27.142863126Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 570.313324ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:27.556352859Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 1.170623522s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:27.58257895Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 513.07223ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:27.863983999Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 550.744492ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:28.137537254Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 1.207023199s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:28.385685999Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 1.001437798s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:28.696924993Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 1.39471772s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:28.761881044Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fe9a:3411","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:29.033227749Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2310"}
{"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:29.0384395Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2371"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:29.160373688Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2556"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:29.403221509Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2720"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:29.591155983Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2435"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 1.250911402s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:29.601988475Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 1.006430519s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:29.707218274Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:29.895176209Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2470"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:30.105422061Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2758"}
{"msg":"wait for service Some(\"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040\") failed: Property not found. retry in 681.172408ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:30.167047385Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:30.292018114Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2785"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:30.511512132Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2664"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:30.696831765Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2464"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:30.707847511Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3843","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:30.707884932Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3859"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:30.707894702Z","hostname":"sock","pid":1618,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"}
{"msg":"wait for service Some(\"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b\") failed: Property not found. retry in 755.737607ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:30.723562597Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399\") failed: Property not found. retry in 1.081797287s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:30.870902326Z","hostname":"sock","pid":1618,"zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.274747824Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3867"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.282016743Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3923"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.436948559Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3985"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.550900991Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3966","rack_subnet":"fd00:1122:3344::"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.574665124Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:4008"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.671052407Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:4035"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.791161572Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.823171627Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.855132381Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:589"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.855212473Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:627"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.855624892Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:630","uptime":"2"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.855635313Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.2.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.935375438Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.93543747Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.93544778Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.93545429Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.93546005Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.9354657Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.935470151Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.935475861Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.935481411Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.935487301Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.935493001Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.935498641Z","hostname":"sock","pid":1618,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.971901198Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_internal_dns_40c6fa81-4587-4a16-bb09-2fff99501399"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.97199359Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_internal_dns_c139c1b3-e02d-4faa-8e97-39cd77d3b040"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.97200154Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_internal_dns_d2bb6497-a52a-4f87-9f06-d2763d7b8a3b"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.972059291Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.972367949Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.972690616Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"b180ff6d-d0a5-4702-a324-e6057a00422e","remote_addr":"[fd00:1122:3344:101::1]:39432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":9498170,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:31.974375775Z","hostname":"sock","pid":1618,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.018984383Z","hostname":"sock","pid":1618,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.064110153Z","hostname":"sock","pid":1618,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:545"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.109210513Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:581"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.109220894Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.140313697Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.140967262Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.140973002Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.169158108Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.169287361Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:57395"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.169432435Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.169441555Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.192154224Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.192188734Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.210385188Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.210437049Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.228151821Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.228209673Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.228394597Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.228554631Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.239962366Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.272287849Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.274294125Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.274299625Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.319350834Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 244, 209]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.319403355Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"ntp"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.320546962Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.320555932Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.3208885Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.322986888Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.347185992Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:32.36517632Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value=\"fd00:1122:3344:100::/56\"/>\n        <propval type=\"boolean\" name=\"boundary\" value=\"true\"/>\n        <propval type=\"astring\" name=\"server\" value=\"0.pool.ntp.org\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.3.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.3.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:33.529757362Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:33.529901686Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 73.885651ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:33.65811879Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 144.564819ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:33.743273032Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 269.861306ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:33.901859112Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 509.197493ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:34.181991481Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 596.065548ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:34.70100322Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:35.239803458Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46700"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:35.239956262Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"9282f81e-3ec5-4e18-bd37-a3f34832d98a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46700","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 1.403205584s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:35.306400688Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 860.020851ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:36.719688189Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305\") failed: Property not found. retry in 1.475416232s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:37.591695622Z","hostname":"sock","pid":1618,"zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.085592728Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.085639209Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.086058928Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.086454718Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"0d8bbc3f-ea51-4dcc-b634-ba08f6ec2ab2","remote_addr":"[fd00:1122:3344:101::1]:57395","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":6917134,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.088217419Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:622"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.117528451Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.117664184Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:47656"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.126223803Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.126357096Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"a560308d-58bb-460a-91e6-3f4e9d06174a","remote_addr":"[fd00:1122:3344:101::1]:47656","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8583,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:39.128752602Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.253574607Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:39.25369854Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49032"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.261683135Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.261805578Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"8e7b06ad-a37e-4451-a3fa-f8bfedd05519","remote_addr":"[fd00:1122:3344:101::1]:49032","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007991,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:43.265288759Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.458910245Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.459037968Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:49527"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.468541699Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.468712453Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"7de99941-8880-44d8-aeec-24af85898836","remote_addr":"[fd00:1122:3344:101::1]:49527","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9568,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:43.471570459Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.586168776Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:43.586260038Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:34386"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_7b9123ca-e648-4548-927b-0f1170fc4305': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:47.594770845Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3574"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:47.594855518Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"7ca5508b-fc01-44ba-ac2e-8d397c3f4dda","remote_addr":"[fd00:1122:3344:101::1]:34386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008509,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:47.597933449Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:47.886970956Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:47.887109889Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64649"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:47.895598657Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"e7aa67fb-43fa-482f-ae45-42b01eacad97","remote_addr":"[fd00:1122:3344:101::1]:64649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8314,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711771363.0129051, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:47.897427189Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:47.897434269Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.036519036Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.036656379Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:44478"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.045231568Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"67c08e04-d222-4736-a6f8-6c5277286726","remote_addr":"[fd00:1122:3344:101::1]:44478","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8454,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711771363.0129051, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.047244765Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:48.047248765Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.396628726Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.396716618Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:64843"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.404365716Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"a9abc24c-ee6b-41a4-8688-22b15750c224","remote_addr":"[fd00:1122:3344:101::1]:64843","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7527,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711771363.0129051, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.405967334Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:48.405973704Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.845108514Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.845279968Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:38302"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.8539704Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"b3ae4591-fe13-4054-ab6f-d6052f851d15","remote_addr":"[fd00:1122:3344:101::1]:38302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8640,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711771363.0129051, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:48.85697413Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:48.85698232Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.092719856Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.092829258Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:51224"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.099993225Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"41555a6b-7efd-4938-aae3-4e06966d8ab8","remote_addr":"[fd00:1122:3344:101::1]:51224","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7057,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711771363.0129051, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.103915866Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:49.103942107Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.357081258Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.357235061Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:37980"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.364929291Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"a78b4396-8d3d-461a-9862-44ba3accbc04","remote_addr":"[fd00:1122:3344:101::1]:37980","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7651,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 2139029761, ip_addr: ::, stratum: 10, ref_time: 1711771363.0129051, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:49.368227797Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:49.368235448Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:652","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.231829906Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:602"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.231938608Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:48081"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.234783195Z","hostname":"sock","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.249770103Z","hostname":"sock","pid":1618,"uri":"/timesync","method":"GET","req_id":"8d27b9af-7838-453c-a8d2-0c3bac40e8eb","remote_addr":"[fd00:1122:3344:101::1]:48081","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":17659,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 3454618057, ip_addr: 205.233.73.201, stratum: 4, ref_time: 1711771369.4840558, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.251993065Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:630"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.252002985Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1210"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.280516789Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.280703083Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:35733"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.280922548Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.280936429Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.302379808Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.302438969Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.320852718Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.320902859Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.339256176Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.339313007Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.390657382Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.390704183Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.440631115Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.440688096Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.495102353Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.495146564Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.547533413Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.547579604Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.598319785Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.598387757Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.598625322Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.598767465Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.630691618Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.63074897Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.63075897Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.630788251Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.630798871Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.630921364Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"45cadaf9-55bf-48e1-bff0-235f39802ebd","remote_addr":"[fd00:1122:3344:101::1]:35733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\", TimeNotSynchronized), (\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\", TimeNotSynchronized), (\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\", TimeNotSynchronized), (\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\", TimeNotSynchronized), (\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\", TimeNotSynchronized)]","latency_us":350132,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.63120732Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"104.157883ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"45cadaf9-55bf-48e1-bff0-235f39802ebd\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"45cadaf9-55bf-48e1-bff0-235f39802ebd\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.736354988Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.737090094Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.737102805Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.75622332Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.756268631Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.77383381Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.77386314Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.791424459Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.79146176Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.809185512Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.809213883Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.827086649Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.8271276Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.845012206Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.845052337Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.863107797Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.863146298Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.880661386Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.880704047Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.880902921Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.881067365Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.913360307Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.913406898Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.913414738Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.913423738Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.913430858Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:50.913578452Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"7c0609c5-673e-457e-9d96-899c042e76c2","remote_addr":"[fd00:1122:3344:101::1]:35733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\", TimeNotSynchronized), (\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\", TimeNotSynchronized), (\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\", TimeNotSynchronized), (\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\", TimeNotSynchronized), (\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\", TimeNotSynchronized)]","latency_us":176720,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:50.913783767Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"146.943995ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7c0609c5-673e-457e-9d96-899c042e76c2\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7c0609c5-673e-457e-9d96-899c042e76c2\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.062140319Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.06301891Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.06302921Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.08362795Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.083680511Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.103190635Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.103245386Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.121237575Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.121286166Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.139970571Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.140022312Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.158077342Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.158126583Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.176200504Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.176252805Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.194341786Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.194391197Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.212642422Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.212700923Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.212892008Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.213022901Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:51.244534334Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:51.244568255Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:51.244581285Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:51.244587745Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:51.244592945Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.244701698Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"3b855562-9fd1-4067-ae0b-6583f362413a","remote_addr":"[fd00:1122:3344:101::1]:35733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\", TimeNotSynchronized), (\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\", TimeNotSynchronized), (\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\", TimeNotSynchronized), (\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\", TimeNotSynchronized), (\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\", TimeNotSynchronized)]","latency_us":182342,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:51.244893692Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"204.670347ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3b855562-9fd1-4067-ae0b-6583f362413a\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:02:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3b855562-9fd1-4067-ae0b-6583f362413a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.450272792Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.450775673Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.450793904Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.469199212Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.469249883Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.487064078Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.487102549Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.504715009Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.50474643Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.522275437Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.522306848Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.539954259Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.539986559Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.558002459Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.55804415Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.001796262Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.001821873Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.003470391Z","hostname":"sock","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.019231478Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.019284899Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.019517524Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.019642597Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.100637812Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.100703624Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.100707854Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.100713254Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.100719124Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.192647824Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.196963644Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.201278955Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.205728358Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.210340455Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.214929902Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.219242502Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.223731687Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.228268853Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:51.232355298Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService6\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::4\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::4\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.770046088Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.770215732Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService7\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::6\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.775859273Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.77612713Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::3\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::3\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.782891327Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.782996499Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService9\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::7\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.815251669Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.815389903Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService8\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::5\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.862290854Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:02:54.862477828Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 46.112663ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:54.920018867Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 52.621711ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:54.9283075Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 38.692611ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:54.935125729Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 37.694709ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:54.953249871Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 61.832193ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:54.973800269Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 101.579044ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:54.996101798Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 157.592661ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:55.338586809Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 141.23674ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:55.56076527Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 80.704592ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:55.767220904Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 171.14684ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:55.893079633Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 265.73059ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:55.930082944Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 241.97258ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:56.120191148Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 70.808704ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:56.307947578Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 162.70869ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:56.422452333Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 529.117051ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:56.962847508Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 477.696854ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.055240539Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 307.180874ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.25425965Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 102.234274ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.350772207Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 586.928021ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.403393291Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 482.890651ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.479534163Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 290.215885ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.516075814Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 1.061164262s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.57692542Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 660.713525ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.605358672Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 211.229894ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.815497831Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 1.154223512s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:57.973254293Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 923.040293ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:58.002674927Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 436.618194ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:58.038685745Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 567.149346ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:58.278793333Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 960.93043ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:58.485642718Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 794.814035ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:58.646783008Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 1.182739316s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:58.855477773Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 1.351637787s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:58.9339821Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 1.02484119s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:59.137670541Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 833.203283ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:59.45131615Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 1.32573895s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:02:59.458582689Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 578.137268ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.048455387Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 648.136636ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.175389951Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 602.480689ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.298576668Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 512.855204ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.310174568Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 1.300175889s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.641492308Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 571.185164ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.798171095Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 728.370551ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.834614142Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 1.081801877s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.847268557Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 1.203547515s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:00.911521522Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 986.377937ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:01.378991661Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 855.857253ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:01.570531539Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 596.90569ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:01.93670486Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 1.059052218s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:01.950359798Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 1.444674116s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:02.123131769Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 675.648296ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:02.373735581Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 1.030123743s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:02.434776142Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 634.205194ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:02.541890464Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9\") failed: Property not found. retry in 1.423794621s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:03.019484279Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9\") failed: Property not found. retry in 1.120219448s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:03.058874186Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d\") failed: Property not found. retry in 1.428834987s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:03.185717568Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 575.649044ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:03.474824936Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7\") failed: Property not found. retry in 789.868263ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:03.577778883Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 1.222848583s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:04.06111292Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:05.110524692Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54714"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:05.110599784Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"f818b530-777c-4941-99cc-a5883104f18b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54714","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 630.584255ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:05.294811501Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 887.052587ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:05.93676116Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 1.398857949s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:07.15407517Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 829.373291ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:08.562549698Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 779.106953ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:09.423163786Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 1.39810567s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:10.212864454Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 859.11637ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:11.62038116Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 857.404417ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:12.489152678Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba\") failed: Property not found. retry in 1.392581888s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:13.356055632Z","hostname":"sock","pid":1618,"zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.770253415Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_2b837ec5-9f47-45f0-a0ad-0b45cda5746d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.770324286Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_9cda62fc-1c62-4db5-bea1-f6915c6e72b7"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.770331327Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_10f37e08-3cf3-4a6d-9b58-c531ed2b12d9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.770338597Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_29c04c78-dae0-44d9-b21b-b3adb1a02fa9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.770357667Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_cockroachdb_3ce5b6ed-8c6a-4176-a669-8e3224034dba"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.770380588Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.771263908Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.772246001Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"9274cfdc-b576-42ae-bf11-7a26100a7f41","remote_addr":"[fd00:1122:3344:101::1]:35733","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":23895647,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.804262006Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:54718"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:14.834322275Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3408"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:15.010877864Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3426"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:26.97946589Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3449"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:26.979597883Z","hostname":"sock","pid":1618,"uri":"/cockroachdb","method":"POST","req_id":"8bfe0265-849d-458f-8da5-02ca947bfd0d","remote_addr":"[fd00:1122:3344:101::1]:54718","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":12175265,"response_code":"204"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.013971323Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.01425452Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::1]:40623"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.014660749Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.014667399Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.037810268Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.037856929Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.058593802Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.058634843Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.07829142Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.078339251Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.097559149Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.09760206Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.116022398Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.116066679Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.134612611Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.134653002Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.153346277Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.153390408Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.171020688Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.17107149Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.289283301Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.289338972Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.412314684Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.412369555Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.504180792Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.504228083Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.605803537Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.605853578Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.7167829Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.716841661Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.827861094Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.827911515Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.937143997Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:27.937195179Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.033467599Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.03352187Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.16327949Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.163346462Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.299541091Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.299594682Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.451482488Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.451536599Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.559295086Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.559362118Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.559608924Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.559779848Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672315987Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672373718Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672382818Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672389398Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672405879Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672413569Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672419559Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672425529Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672432499Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.67243826Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.67244401Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.67244945Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.67245505Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.67246094Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.67246666Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.67247372Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672479191Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672483361Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672489171Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.672665495Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"bb43fe80-b4d5-4878-9b7e-b0aea392757c","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized)]","latency_us":1658262,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:28.672937531Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"96.761099ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb43fe80-b4d5-4878-9b7e-b0aea392757c\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bb43fe80-b4d5-4878-9b7e-b0aea392757c\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.771080015Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.771555046Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.771569037Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.790920467Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.790960088Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.809198922Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.809236873Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.825622995Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.825662266Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.841935234Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.841985505Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.859852411Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.859892932Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.875712401Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.875746801Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.89286579Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.892904241Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.909868445Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.909909366Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.927704731Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.927744871Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.945175777Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.945219358Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.961522927Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.961563059Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.978019611Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.978057702Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.994637668Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:28.994690039Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.012539875Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.012587946Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.03079103Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.030833011Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.049072595Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.049113776Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.065272952Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.065309823Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.082245687Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.082282568Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.101549316Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.101587117Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.119487324Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.119533375Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.11975785Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.119931514Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212762214Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212816806Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212825056Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212832296Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212838156Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212863617Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212868367Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212875697Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212881217Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212886968Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212892568Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212897838Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212905048Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212910738Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212916288Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212922408Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212929349Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212935078Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.212942679Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.213051521Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"9c5378fb-fae5-45e9-a0ad-3a39d40454ef","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized)]","latency_us":441696,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.213225285Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"87.738956ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9c5378fb-fae5-45e9-a0ad-3a39d40454ef\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9c5378fb-fae5-45e9-a0ad-3a39d40454ef\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.301607792Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.302085223Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.302092853Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.32215654Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.322204461Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.339592336Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.339643847Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.358695861Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.358745622Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.376318181Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.376378012Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.394829162Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.394877993Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.412867562Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.412915893Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.432844207Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.432893858Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.451325257Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.451372398Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.472307285Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.472361326Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.491319957Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.491367998Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.510854752Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.510900143Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.531860511Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.531906022Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.550873913Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.550917534Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.567989542Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.568044323Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.585634912Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.585679693Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.60359156Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.603637191Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.621780204Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.621825565Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.640396737Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.640464318Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.661733423Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.661786035Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.682756923Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.682814404Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.683038599Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.683188893Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775052751Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775095882Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775103332Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775108252Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775114322Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775121432Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775127622Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775144463Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775150653Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775155583Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775161463Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775167383Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775174483Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775180544Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775187144Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775194084Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775201114Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775208094Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775214934Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.775330417Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"6bb0d3f3-3f99-4018-8ffe-cb5c099ac965","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized)]","latency_us":473428,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:29.775516422Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"153.590824ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6bb0d3f3-3f99-4018-8ffe-cb5c099ac965\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6bb0d3f3-3f99-4018-8ffe-cb5c099ac965\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.929709439Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.930219411Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.930225871Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.953321709Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.95336006Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.972443324Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.972483205Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.993504704Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:29.993541295Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.014726418Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.014762949Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.034355965Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.034387105Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.051295179Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.051328649Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.070981617Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.071017238Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.089674762Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.089706513Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.109299969Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.10934275Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.12785737Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.127897551Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.145734186Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.145774967Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.167045073Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.167077623Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.186002224Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.186040225Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.203802628Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.203859279Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.221586872Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.221617873Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.240906092Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.240937742Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.259502044Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.259534985Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.277917553Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.277949694Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.298658745Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.298690466Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.316981752Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.317018993Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.317272969Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.317426412Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410650142Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410693973Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410699363Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410703823Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410708063Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410711413Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410715604Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410719664Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410723944Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410744154Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410748524Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410752694Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410756654Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410760754Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410765735Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410770105Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410774995Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410779135Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.410782475Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.410880647Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"f2d6bbbf-2514-4d3a-a6e5-c43c298a6b0e","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized)]","latency_us":480836,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:30.411087972Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"182.529576ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2d6bbbf-2514-4d3a-a6e5-c43c298a6b0e\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2d6bbbf-2514-4d3a-a6e5-c43c298a6b0e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.594298456Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.595363741Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.595382571Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.617880205Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.617916986Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.637961952Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.638000413Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.658214364Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.658249174Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.677531003Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.677566074Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.69543437Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.695475381Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.714842481Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.714875182Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.733202629Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.733233939Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.751390522Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.751420343Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.770068537Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.770106168Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.788195678Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.788225749Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.806134785Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.806164736Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.824894292Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.824936323Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.843329961Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.843362752Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.86177817Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.861809011Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.880591858Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.880623479Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.899948829Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.89998979Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.919159556Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.919188916Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.938046385Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.938080906Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.956822902Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.956871283Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.974989605Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.975030066Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.975242371Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:30.975408835Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.067960489Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.067998709Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.06800566Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.06801066Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.06801526Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.06802029Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.06802452Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.06802798Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.06803305Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068037941Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068054921Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068060221Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068064411Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068068681Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068072811Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068077941Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068082122Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068086351Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068091342Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.068184414Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"abbd8359-ce32-41b0-bfba-321fc02352d3","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized)]","latency_us":473578,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.068366638Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"181.354472ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"abbd8359-ce32-41b0-bfba-321fc02352d3\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"abbd8359-ce32-41b0-bfba-321fc02352d3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.250750583Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.251210123Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.251216293Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.272208222Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.272247223Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.29233122Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.292361921Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.31251479Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.312545741Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.331046751Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.331078472Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.349823288Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.349850399Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.371787839Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.37182614Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.390489265Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.390524395Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.411109285Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.411140935Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.431899298Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.431926679Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.45174557Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.451779671Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.471331786Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.471362507Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.490454501Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.490482092Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.50888715Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.508916451Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.528829044Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.528859305Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.548640415Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.548681056Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.567168497Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.567201177Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.586378544Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.586408854Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.605906158Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.605934589Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.624671055Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.624704376Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.64378891Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.643823461Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.644055336Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.644207739Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740247695Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740300896Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740307426Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740313096Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740318686Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740324507Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740330667Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740335097Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740340767Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740346427Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740352187Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740357997Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740381758Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740389578Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740396908Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740402438Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740407788Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740413358Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740418939Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:31.740534031Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"826c7961-b768-49b6-9a9a-520b0a31fa25","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized)]","latency_us":489507,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:31.740762577Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"352.499342ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"826c7961-b768-49b6-9a9a-520b0a31fa25\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"826c7961-b768-49b6-9a9a-520b0a31fa25\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.094886047Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.095403169Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.09541339Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.116972261Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.117036973Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.135031672Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.135077022Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.154986806Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.155038207Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.173790483Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.173833284Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.192137641Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.192178391Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.210666302Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.210719383Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.230976665Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.231028266Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.247808816Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.247851917Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.26600558Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.266054231Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.285063253Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.285105784Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.302145531Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.302190092Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.322994156Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.323041567Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.341894096Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.341935287Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.359533066Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.359573377Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.379258425Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.379300546Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.397612203Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.397653284Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.41514192Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.415188021Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.434698726Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.434740137Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.455712545Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.455761656Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.473405616Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.473464198Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.473665562Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.473787545Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566657367Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566722948Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566731568Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566738679Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566744399Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566751819Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566757939Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566763379Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566769039Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566775949Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.56678184Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.56678933Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.56679481Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.56679909Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.56681537Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.56682101Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566828151Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566836651Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.566843501Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.566961324Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"e0bcca0f-d230-4e0a-bbe7-700b8c19fd00","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized)]","latency_us":471760,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:32.567880475Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"300.769173ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e0bcca0f-d230-4e0a-bbe7-700b8c19fd00\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e0bcca0f-d230-4e0a-bbe7-700b8c19fd00\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.86933751Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.869807451Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.869817761Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.892490719Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.89254487Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.912597907Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.912646498Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.932368567Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.932419148Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.949729091Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.949776302Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.965663412Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.965712083Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.983510557Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:32.983556779Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.00383507Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.003883801Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.024120863Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.024168454Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.043256838Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.043316889Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.060811596Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.060859087Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.080626207Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.080681649Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.097757276Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.097804387Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.116714138Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.116761299Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.134307647Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.134348808Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.153691248Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.153736239Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.171779219Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.17184383Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.190083475Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.190139796Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.208213567Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.208258688Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.228806376Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.228851617Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.249048977Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.249114269Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.249327504Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.249457477Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345683496Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345752048Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345761518Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345768118Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345776478Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345782978Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345789439Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345796949Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345803339Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345811089Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345815209Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345819749Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.34582621Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.34583102Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.3458359Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.34585718Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.34586368Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345869791Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.345876201Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.345931422Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"ce1b48db-c088-4e45-b22e-182678b377f6","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized)]","latency_us":476307,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:33.346159097Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"217.740746ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ce1b48db-c088-4e45-b22e-182678b377f6\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ce1b48db-c088-4e45-b22e-182678b377f6\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.56457511Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.565656255Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.565664576Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.587857372Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.587901723Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.608510683Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.608567464Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.628264803Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.628311764Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.645989725Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.646045717Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.663496683Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.663535333Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.682193068Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.682237669Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.70036373Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.700416852Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.71666421Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.716709121Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.73555466Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.735605381Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.754002999Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.7540465Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.77166088Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.771707841Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.789597137Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.789638868Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.807926443Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.807961564Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.825766888Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.825806169Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.842993519Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.84303234Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.860824844Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.860866295Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.877917992Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.877959443Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.895575653Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.895615644Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.914071173Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.914117354Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.931692024Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.931746985Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.93196686Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:33.932093693Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.026901659Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.02695237Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.026977311Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.026983251Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.026990431Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.026996182Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027001982Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027008712Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027015192Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027022592Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027028782Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027034672Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027040183Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027047263Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027053463Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027060633Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027064983Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027073413Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.027077543Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.027163135Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"2b700a8a-1408-493f-ac20-aa796ca7a589","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized)]","latency_us":462289,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.02738203Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"251.426091ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2b700a8a-1408-493f-ac20-aa796ca7a589\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2b700a8a-1408-493f-ac20-aa796ca7a589\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.279837446Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.280360248Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.280373988Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.302699488Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.302749749Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.320665226Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.320707497Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.341813528Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.341858049Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.36166669Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.361715181Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.379837383Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.379883824Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.399710136Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.399757417Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.418686297Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.418734978Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.437743291Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.437801562Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.458455793Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.458504934Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.479773329Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.47983135Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.499962979Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.5000144Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.519041253Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.519093634Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.538341012Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.538388523Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.557045747Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.557089838Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.575935317Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.575979438Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.595744018Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.595791629Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.614780511Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.614842352Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.633366673Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.633407274Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.653358819Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.65342157Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.676034526Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.676095928Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.676304953Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.676462396Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772728287Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772775778Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772783818Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772800448Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772806379Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772812049Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772817869Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772824639Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772830029Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772835089Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772840369Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772845529Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.772851169Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.77285476Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.77285851Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.77286384Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.77286749Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.77287074Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.77287455Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:34.772962982Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"d7d4248a-eb9d-4ed9-b74c-7534cca75f6a","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized)]","latency_us":492795,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:34.773205938Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"298.580596ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d7d4248a-eb9d-4ed9-b74c-7534cca75f6a\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d7d4248a-eb9d-4ed9-b74c-7534cca75f6a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.072650997Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.073116077Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.073123478Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.095270463Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.095319404Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.117001439Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.11704617Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.136878941Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.136927403Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.141949279Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36678"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.142093243Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"9586cce0-b3b1-43c6-9d58-240c3a2a718f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36678","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":77,"response_code":"200"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.156019477Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.156066488Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.176142405Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.176184746Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.19569094Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.195736031Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.214929108Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.214975959Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.233608473Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.233659564Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.25326308Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.253305821Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.273411819Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.27345842Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.292136995Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.292177326Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.311593178Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.311666639Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.329985316Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.330029047Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.349833957Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.349875678Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.368875451Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.368917432Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.390132595Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.390179767Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.410296525Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.410339976Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.429991003Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.430041544Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.45005662Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.450098771Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.469148874Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.469202946Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.469425341Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.469565914Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564153455Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564214657Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564241847Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564247767Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564255118Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564260858Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564265898Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564269938Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564276988Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564282628Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564288588Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564294099Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564300939Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564306639Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564313729Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564319579Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564325139Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.56433644Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.5643422Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:35.564462783Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"811a8c7f-da35-408d-ad19-753def14dfbc","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized)]","latency_us":491535,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:35.564674168Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"809.373835ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"811a8c7f-da35-408d-ad19-753def14dfbc\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"811a8c7f-da35-408d-ad19-753def14dfbc\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.375230621Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.375733492Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.375741403Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.397408737Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.397458738Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.417535865Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.417581006Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.43752547Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.437580662Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.456568324Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.456616065Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.475545575Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.475592887Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.493996455Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.494044986Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.512487605Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.512535056Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.532548412Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.532595813Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.551144125Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.551200216Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.570535396Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.570581537Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.589448756Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.589497047Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.609498543Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.609545294Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.627526843Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.627575654Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.645449169Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.645505271Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.663982371Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.664031502Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.682453091Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.682501372Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.700443149Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.700495461Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.71892981Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.718972731Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.73783651Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.737888051Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.757025166Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.757090158Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.757294852Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.757433966Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854142286Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854196698Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854203508Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854210758Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854225378Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854232608Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854238179Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854242009Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854245699Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854249469Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854254439Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854258049Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854261619Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854265279Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854269219Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854274229Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.8542795Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.85428322Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.85428818Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:36.854349221Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"3af180c1-7c0a-48fd-bdc9-0eefcacbf7b8","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized)]","latency_us":478812,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:36.854564256Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"863.990446ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3af180c1-7c0a-48fd-bdc9-0eefcacbf7b8\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3af180c1-7c0a-48fd-bdc9-0eefcacbf7b8\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.719656119Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.720275434Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.720283334Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.74376155Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.743815772Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.767045432Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.767098604Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.788681806Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.788732897Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.808888005Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.808935297Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.827786505Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.827837877Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.847066164Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.847112735Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.865879192Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.865923693Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.885181691Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.885244153Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.903882526Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.903927557Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.922410377Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.922457248Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.941536792Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.941583994Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.960991955Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.961039397Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.980668063Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:37.980734475Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.00201458Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.002083252Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.028041526Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.028101337Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.049794592Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.049838463Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.070028713Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.070074674Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.090995211Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.091048572Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.109931631Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.109980903Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.12876752Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.128822101Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.129035166Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.12918822Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.22587647Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.225943231Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.225951642Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.225957442Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.225963072Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.225989112Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.225995223Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226002653Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226008883Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226013023Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226016693Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226022153Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226025883Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226029664Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226033404Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226037184Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226040714Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226044424Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226049654Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:38.226118095Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"3951fd22-e991-4e39-884e-ac4d342548c5","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized)]","latency_us":506050,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:38.226376072Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.059553953s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3951fd22-e991-4e39-884e-ac4d342548c5\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3951fd22-e991-4e39-884e-ac4d342548c5\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.287063737Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.288256274Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.288264765Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.310992594Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.311046355Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.333384905Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.333433716Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.353326319Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.35337441Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.373071858Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.3731327Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.392779367Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.392838068Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.4122634Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.412314032Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.433538606Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.433583467Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.452672261Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.452724572Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.473297751Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.473341192Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.492433686Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.492482407Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.511614653Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.511658604Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.53040388Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.530448501Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.550570699Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.55061628Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.569636653Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.569680194Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.589195348Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.589239699Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.609417219Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.60946351Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.628346709Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.62839455Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.648351895Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.648449947Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.671533245Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.671586906Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.693034845Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.693099766Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.693301531Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.693468985Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789280255Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789336856Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789343316Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789349276Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789356116Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789361717Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789368687Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789386437Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789393747Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789399608Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789405578Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789411438Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789417018Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789424008Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789429218Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789434638Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789441538Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789445149Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789448879Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:39.789541611Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"f4ada595-bb4c-479d-8df7-96a6badb2412","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized)]","latency_us":502140,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:39.789796417Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.470099039s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f4ada595-bb4c-479d-8df7-96a6badb2412\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f4ada595-bb4c-479d-8df7-96a6badb2412\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.260378631Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.261564438Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.261578009Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.283951119Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.28399267Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.304360754Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.304403835Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.324529424Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.324576475Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.343381642Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.343423433Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.363391638Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.363438679Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.382900522Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.382941373Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.405872977Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.405929078Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.42447379Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.424516121Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.442320495Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.442373796Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.461369798Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.461410199Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.479515271Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.479553782Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.496961027Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.496999788Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.515112339Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.51515343Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.532924974Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.532981115Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.552721584Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.552766215Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.571713106Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.571750187Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.590551905Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.590592836Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.609361492Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.609398033Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.629695426Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.629733957Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.649587629Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.64963546Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.649839585Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.649968197Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746588136Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746651418Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746659838Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746665228Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746670808Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746677858Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746683378Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746689308Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746694899Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746722389Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.7467301Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.74673436Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.74674156Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.74674744Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.74675468Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.74676173Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.74676759Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.74677343Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.746777321Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:41.746864123Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"37cb41a8-44a9-478f-97db-78a1e1feb554","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized)]","latency_us":486182,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:41.747056367Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.684111027s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"37cb41a8-44a9-478f-97db-78a1e1feb554\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"37cb41a8-44a9-478f-97db-78a1e1feb554\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.432973362Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.434081737Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.434106528Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.456226393Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.456284414Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.48057379Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.480629181Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.500572175Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.500620646Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.522074355Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.522119666Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.539516641Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.539572292Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.560892869Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.56093637Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.581777095Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.581824606Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.602461446Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.602513597Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.623317332Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.623363853Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.644109315Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.644152256Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.664101641Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.664152002Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.683340148Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.683386539Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.702973495Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.703023476Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.722646843Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.722691414Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.741345058Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.74139166Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.7611628Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.761209161Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.782983107Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.783029129Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.803461143Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.803509154Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.82225345Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.822312562Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.840940795Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.840996917Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.841200171Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.841332605Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935623959Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935690341Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935699661Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935708081Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935716111Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935722821Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935729281Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935737371Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935743822Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935751852Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935759752Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935775043Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935779983Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935784603Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935789113Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935795153Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935799783Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935805753Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.935813333Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:43.935874765Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"a79afb21-f362-451b-ab78-8820b7c66fec","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized)]","latency_us":502592,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:43.936075819Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.411344434s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a79afb21-f362-451b-ab78-8820b7c66fec\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a79afb21-f362-451b-ab78-8820b7c66fec\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.348856788Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.349336589Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.349344219Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.371760951Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.371810642Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.391783917Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.391834928Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.416122043Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.416176685Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.437987162Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.438035313Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.457005355Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.457045826Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.476453858Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.476502189Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.498072881Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.498113532Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.518360993Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.518413364Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.538591204Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.538632515Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.558311173Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.558359944Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.578218656Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.578259357Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.598475817Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.598517028Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.618911533Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.618950834Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.638824476Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.638864647Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.660402228Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.66045968Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.680084857Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.680131308Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.70001177Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.700071682Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.721829388Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.721880449Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.740290378Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.740333609Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.759067425Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.759129376Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.759353422Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.759486914Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853639015Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853690986Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853697406Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853702886Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853708206Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853713887Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853719727Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853724977Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853732037Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853737787Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853744937Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853752027Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853777468Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853784718Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853788528Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853792228Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853797608Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853801549Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.853805219Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:45.853886981Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"8fc046a3-666f-4fa7-91d9-b0fa42d4831a","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized)]","latency_us":504729,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:45.854098935Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"1.111513595s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8fc046a3-666f-4fa7-91d9-b0fa42d4831a\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8fc046a3-666f-4fa7-91d9-b0fa42d4831a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:46.966521024Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:46.967013226Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:46.967021056Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:46.988468045Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:46.988522856Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.008293866Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.008342787Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.02692234Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.026970681Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.047239162Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.047287464Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.06562138Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.065662861Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.083422505Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.083476216Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.10341331Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.103462731Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.122921654Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.122970075Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.142228913Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.142273434Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.161073652Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.161114723Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.180096754Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.180144946Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.197550461Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.197593452Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.216707426Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.216752187Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.234206024Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.234252935Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.253673397Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.253717438Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.272348752Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.272395282Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.290179896Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.290228448Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.309351353Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.309392054Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.327892364Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.327943325Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.352483246Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.352539068Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.352781493Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.352916967Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448809108Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448855209Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448862269Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448868129Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44887295Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44887764Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44888304Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44888722Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44889154Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44889558Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44890059Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448904621Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448909181Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448913701Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448929781Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448934511Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448938671Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448943961Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.448947351Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:47.449113345Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"a7219449-67ef-4640-9d50-4c8a4b434498","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized)]","latency_us":482287,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:47.44932266Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"2.229541294s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a7219449-67ef-4640-9d50-4c8a4b434498\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a7219449-67ef-4640-9d50-4c8a4b434498\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.680164377Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.680647968Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.680654759Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.702927527Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.702963008Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.723865814Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.723905735Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.744593677Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.744633488Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.765983185Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.766024566Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.786068872Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.786115863Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.806739282Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.806776003Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.826797349Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.8268354Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.846016916Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.846063847Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.865554711Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.865595182Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.88483554Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.884878721Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.904347864Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.904426886Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.925740812Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.925784543Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.946004613Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.946046914Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.964855792Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.964897443Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.987514499Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:49.98755656Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.006120502Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.006176074Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.02664679Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.026697921Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.047074795Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.047114116Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.069539948Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.069578059Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.089750949Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.08979821Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.090043086Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.090199679Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.18689622Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.186976611Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.186986132Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.186992412Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.186998422Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187004242Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187018332Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187024063Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187029503Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187042883Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187047243Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187051193Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187057053Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187061383Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187065484Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187069524Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187094454Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187100424Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187105854Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:50.187154686Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"c78b9f25-7791-4d30-a4a7-c1d97309a2c3","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized)]","latency_us":506691,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:50.187381261Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"3.590353665s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c78b9f25-7791-4d30-a4a7-c1d97309a2c3\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c78b9f25-7791-4d30-a4a7-c1d97309a2c3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.779043726Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.779537988Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.779547798Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.802777178Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.80283825Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.823354427Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.823425539Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.843946676Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.844015318Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.862701413Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.862752554Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.881531211Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.881586382Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.900606925Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.900665236Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.919641318Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.919698149Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.939647783Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.939710225Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.959486195Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.959546857Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.979302057Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.979363038Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.999066816Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:53.999117098Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.021256823Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.021318674Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.042345644Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.042411275Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.062616805Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.062670477Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.082119819Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.082178561Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.103245911Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.103303612Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.12425665Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.124321512Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.142926914Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.142992446Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.163174586Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.163238467Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.183251543Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.183321374Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.183519659Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.183656002Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280155108Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.28024098Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280274231Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280282041Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280288111Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280293521Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280300731Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280306832Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280313682Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280325642Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280332862Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280338592Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280349333Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280354023Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280358543Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280363143Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280367523Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280378543Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.280384744Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:54.280453265Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"1fdf3f3b-6b1d-48e9-8574-a67b724d6a27","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized)]","latency_us":501112,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:54.28066916Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"4.72584831s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1fdf3f3b-6b1d-48e9-8574-a67b724d6a27\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1fdf3f3b-6b1d-48e9-8574-a67b724d6a27\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.839313611Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.83979831Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.839807591Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.843114277Z","hostname":"sock","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.860569251Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.860626037Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.880801352Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.88085Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.899529377Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.899582217Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.921328288Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.921389967Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.943422751Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.943473433Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.963351949Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.965702649Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.982131364Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:58.984094012Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.001312285Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.001429403Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.0213531Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.021468495Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.041089674Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.041147876Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.061321369Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.061382241Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.080842207Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.080895084Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.099717859Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.099769076Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.11988926Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.119939971Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.139579342Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.139627728Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.159310583Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.159362475Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.178122794Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.17818348Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.198203752Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.198256394Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.218051463Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.218105436Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.23719352Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.237249311Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.237482839Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.237630433Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333665331Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333751621Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333758268Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333785024Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333791053Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333800756Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333807272Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333814687Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333831601Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333841144Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333848129Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333855732Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333863561Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333870274Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333875936Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333885124Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333891255Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333896758Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.333919493Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:03:59.333926618Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"4cf20183-8a51-4631-9e63-e38868c4b20a","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized)]","latency_us":492930,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:03:59.33413772Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"2.104668167s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4cf20183-8a51-4631-9e63-e38868c4b20a\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:03:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4cf20183-8a51-4631-9e63-e38868c4b20a\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.445721711Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.446239228Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.44624879Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.467406586Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.469451179Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.487326414Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.487389426Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.506665467Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.506725318Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.525257072Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.525309405Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.543355441Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.543413793Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.561217161Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.561344038Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.579254119Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.57938457Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.597322812Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.597449298Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.615598643Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.615659029Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.633564856Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.633613401Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.651680438Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.651729048Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.669153016Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.66920518Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.687400453Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.689463717Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.708103688Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.708155224Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.726047087Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.726104772Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.744019529Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.744070071Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.764120094Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.764175126Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.782271187Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.782328058Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.801394547Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.801441985Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.819708953Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.819767708Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.820041757Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.820175429Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.911860919Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.911958047Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.911968219Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.911975437Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912003545Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912013782Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912032102Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912042058Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912049633Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912067586Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.91207729Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912086731Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912093397Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912100812Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912106681Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.91211241Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912120332Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912126369Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.91213596Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:01.912263692Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"f0452375-ae93-4314-bfeb-da783be42cc3","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized)]","latency_us":464936,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:01.912496572Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"5.954249529s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f0452375-ae93-4314-bfeb-da783be42cc3\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:04:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f0452375-ae93-4314-bfeb-da783be42cc3\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:05.024731426Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64805"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:05.024816168Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"5e7b6bab-72bd-4954-bef1-a80213cffe08","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64805","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":56,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.883903946Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.885122867Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.885137492Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.90684059Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.906887775Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.926431293Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.92655437Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.944701062Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.944749364Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.962960226Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.963014949Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.980587015Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.980710533Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.997819237Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:07.997874449Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.015848003Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.015897513Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.033586259Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.03364207Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.051351409Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.051401389Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.069527708Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.069579009Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.087650754Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.087728952Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.105787143Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.105860158Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.12480298Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.124853306Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.142593241Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.142647225Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.161199374Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.161266501Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.18028897Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.180338743Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.197949681Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.198017839Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.217996168Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.218059731Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.23646604Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.236517717Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.254476601Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.254539818Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.25473667Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.254866422Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346027197Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346108968Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346116402Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346142588Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346150219Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346158806Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346165388Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346173563Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346180145Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346188357Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346194798Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.34620152Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346209601Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346215967Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346224077Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346230593Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346237511Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346245583Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346259984Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:08.346360833Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"d8db7373-3541-472a-b620-eda658b0f8fa","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized)]","latency_us":460844,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:08.346710056Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"6.18627504s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d8db7373-3541-472a-b620-eda658b0f8fa\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:04:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d8db7373-3541-472a-b620-eda658b0f8fa\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.551607038Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.552151007Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.552164386Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.573074053Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.573147948Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.591381405Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.591440328Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.611310187Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.61370013Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.629472847Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.629532146Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.646893063Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.64694833Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.666074372Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.666122983Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.68404378Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.684123629Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.701863198Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.701920377Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.720057247Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.720120304Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.740442927Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.740505938Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.758650702Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.758705417Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.777235753Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.777288423Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.796580788Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.796633337Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.814602069Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.814649827Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.832953674Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.833000953Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.852005947Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.852073055Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.86907832Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.869136971Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.887334677Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.887383906Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.90773795Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.907790638Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.926173854Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.926257912Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.926538252Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:14.926712677Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019008148Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019071617Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019085633Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019092009Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019099368Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019127063Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019133054Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019139963Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019145672Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019150857Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019157935Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019165052Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019176189Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019181823Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.01918593Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019195305Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.01919928Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019203386Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019207427Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:15.019278716Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"ef0c305c-07e6-45b7-90ba-fef9976c4653","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized)]","latency_us":466055,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:15.019462321Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"7.679561835s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef0c305c-07e6-45b7-90ba-fef9976c4653\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:04:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef0c305c-07e6-45b7-90ba-fef9976c4653\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.721704397Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.722211643Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.722219321Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.745785612Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.745833013Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.765891907Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.765936927Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.785313914Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.785354086Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.805949962Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.805987228Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.827512287Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.827561244Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.846390223Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.846430686Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.865916691Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.865958167Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.884789606Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.884828242Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.905413118Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.905452729Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.926329175Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.926372347Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.947135376Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.947183443Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.966029091Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.966132498Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.986001773Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:22.986067043Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.00567573Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.005727614Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.028176066Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.028223412Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.048718305Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.048757793Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.067894057Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.067936629Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.087788408Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.087842963Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.10723516Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.107279139Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.126624964Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.126673011Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.126904383Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.12705974Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217720094Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217838103Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217844469Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217848867Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.21785331Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217857754Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217880142Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217885157Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.21788963Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.21789353Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217901836Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217910883Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217914455Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217921599Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217926146Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217929352Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217936102Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217939731Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.217946434Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:23.218022645Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"2e7e3ae2-4ec2-4a80-946c-699a470b975d","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized)]","latency_us":494625,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:23.21820999Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"8.732497026s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2e7e3ae2-4ec2-4a80-946c-699a470b975d\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:04:22 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2e7e3ae2-4ec2-4a80-946c-699a470b975d\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:31.975598362Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:31.976337082Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:31.976346327Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:31.998668629Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:31.998743443Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.018071876Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.018123337Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.035986071Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.036025813Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.055132568Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.055203811Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.073557849Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.073593615Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.093550539Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.093590758Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.111196851Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.111237071Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.128932666Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.128967439Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.148202055Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.148262479Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.166490114Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.166535564Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.184414114Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.184462331Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.202981677Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.203088313Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.221730657Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.221779868Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.240429824Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.240470869Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.259128301Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.259235415Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.277017845Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.277066033Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.294732721Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.294777459Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.313756145Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.313809386Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.331565497Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.33160267Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.349724419Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.349762258Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.34999751Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.350176427Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442764603Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.44280171Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442807701Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442812857Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442817788Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.44282212Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442826432Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442831393Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442849796Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442853986Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442861158Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442864449Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442871443Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442875193Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442878343Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442893775Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442897047Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442901331Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.442905578Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:32.442929832Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"ba86f3e2-f67f-42d6-952c-7708bd7e522c","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized)]","latency_us":465592,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:32.443086079Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"12.413310517s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ba86f3e2-f67f-42d6-952c-7708bd7e522c\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:04:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ba86f3e2-f67f-42d6-952c-7708bd7e522c\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:35.137405405Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45866"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:35.137516904Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"797dbaf7-283d-4e95-ab12-57e34e64db08","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45866","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":78,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.89116102Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.891720018Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.891726308Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.914228168Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.914303518Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.933742859Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.933776863Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.953252628Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.953304962Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.973491482Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.973618456Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.991781406Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:44.991816704Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.009485721Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.009535776Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.029014476Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.029049877Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.047795964Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.047836596Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.066614309Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.066655532Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.086273865Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.086314019Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.105292585Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.105334109Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.123206668Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.123247169Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.142179421Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.142225369Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.15985068Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.159883456Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.179000137Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.179029782Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.19870276Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.198736455Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.216748619Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.216788051Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.235723971Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.235773078Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.25463647Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.254667454Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.272670719Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.272707648Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.272951788Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.273077463Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368372125Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368422076Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368439027Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368444952Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368450042Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368453295Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368464339Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.36846823Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368471634Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368474859Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.36847874Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368482921Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368486831Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368489981Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368497696Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368501896Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368505074Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368509181Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368512397Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:45.368531044Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"4a787e8b-b62c-4eef-b523-224fb49eddd3","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized)]","latency_us":475751,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:45.368773507Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"7.523113325s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4a787e8b-b62c-4eef-b523-224fb49eddd3\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:04:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4a787e8b-b62c-4eef-b523-224fb49eddd3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.913915897Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.914404384Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.914412212Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.936750934Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.936791735Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.95647014Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.956516716Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.976057126Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.976100608Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.9948285Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:52.994881413Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.013729636Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.013772143Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.033364926Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.033418638Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.051660556Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.051704338Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.069509619Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.069562664Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.088888548Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.088939118Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.106454841Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.106562422Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.124478346Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.124516016Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.143030393Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.143063113Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.160706462Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.160741076Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.17907706Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.17911788Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.197199918Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.197237803Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.215163039Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.215207731Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.233278029Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.233327164Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.251335375Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.251373589Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.269991289Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.270034255Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.287787169Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.287826807Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.288119324Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.288261414Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378829144Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.37887707Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378885611Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378890973Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378898239Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.37890363Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378909865Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378924874Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378946887Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.3789512Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378955316Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378959263Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378964185Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378969088Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378974835Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.37897985Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378983806Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378988606Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.378993407Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:04:53.379018673Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"0b3da622-9ef0-46ac-9aeb-1f7ffed9286e","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized)]","latency_us":463572,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:04:53.379208437Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"18.910845539s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0b3da622-9ef0-46ac-9aeb-1f7ffed9286e\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:04:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0b3da622-9ef0-46ac-9aeb-1f7ffed9286e\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:05.252061322Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42834"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:05.252180293Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"96dc0c88-4fe6-4f69-95ad-3f7a6c741604","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42834","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":65,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.342712453Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.343236669Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.343245013Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.364817443Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.364858844Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.384154091Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.384202542Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.404302537Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.404342916Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.423738146Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.4237867Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.441644425Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.441690542Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.459504916Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.459542314Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.478282682Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.478323099Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.496328005Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.496358015Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.514591826Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.514630313Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.53311192Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.533144959Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.551095519Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.551136273Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.568836882Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.568867698Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.5864158Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.586466585Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.604729348Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.604780228Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.623589905Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.623634943Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.64106124Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.641107572Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.658640005Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.658676868Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.6762812Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.676399982Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.695147773Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.695194557Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.713214437Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.713254741Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.713585698Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.713734876Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804022481Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804065279Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.80407112Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804075414Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804079792Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804084404Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804088614Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804100887Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804110421Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804115596Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804120705Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804123809Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804127409Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804131505Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804134712Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804137881Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.80414104Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804144172Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804148119Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:12.804206189Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"6f7c7b16-0473-4fa0-97e6-629ebac0fed7","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized)]","latency_us":459930,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:12.804381862Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"18.002048099s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6f7c7b16-0473-4fa0-97e6-629ebac0fed7\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:05:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6f7c7b16-0473-4fa0-97e6-629ebac0fed7\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.856363803Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.857189374Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.857213524Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.879148503Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.879220412Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.897298206Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.897334Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.915508247Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.915543123Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.93454741Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.934595861Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.954242663Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.954287598Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.973079432Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.973121583Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.991401106Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:30.99145191Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.010925039Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.010965906Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.030549978Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.030587319Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.049388277Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.04943656Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.069384579Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.069418977Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.08791561Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.087966153Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.105853456Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.105889185Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.123920836Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.123973862Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.142384069Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.142419639Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.160715762Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.160752457Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.178457361Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.178492631Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.196573562Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.196616013Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.214885421Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.21492805Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.233708193Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.233744082Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.233982559Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.234110887Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325623993Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325673786Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325679936Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325685542Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325689695Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325693923Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325701574Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325772175Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325791675Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325797103Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325803506Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325809947Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325814101Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325817457Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325821563Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325825473Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325829607Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325833826Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.325836892Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:31.325872142Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"0e24c64f-11cd-4929-b60a-e1cd6aff1e87","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized)]","latency_us":467704,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:05:31.326043962Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"28.277222754s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0e24c64f-11cd-4929-b60a-e1cd6aff1e87\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:05:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0e24c64f-11cd-4929-b60a-e1cd6aff1e87\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:35.368314909Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57476"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:35.368413508Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"d00eac66-5b10-4185-b8a9-0e122d614739","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57476","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":65,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.681237233Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.681851322Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.681860134Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.701871387Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.701908898Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.720502218Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.720544369Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.736016638Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.736051026Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.753331401Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.753367252Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.771687672Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.771732449Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.789346955Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.789385712Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.806779251Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.806810611Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.824787381Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.824823813Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.842098684Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.842141435Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.860606408Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.860643862Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.877207453Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.877243472Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.892687352Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.892719462Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.911134491Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.911181752Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.927574393Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.927609663Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.945177874Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.945214504Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.96371922Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.963775958Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.981082936Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.981200792Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.997531471Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:05:59.997576238Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:00.013571267Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:00.013605383Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:00.031192783Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:00.031234672Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:00.031573821Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:00.031714936Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125694905Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125753763Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125761769Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125767066Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125772935Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125778082Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125784373Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125799326Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125805776Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125811607Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125817176Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125825549Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125829824Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125833743Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125837858Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125846877Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125851077Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125856046Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.125861258Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:00.125909542Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"02cd5a37-8e10-4c42-896f-313da530454a","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized)]","latency_us":443070,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:00.126095705Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"12.739579881s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"02cd5a37-8e10-4c42-896f-313da530454a\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:05:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"02cd5a37-8e10-4c42-896f-313da530454a\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:05.480664956Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62631"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:05.480739498Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"1be6d28d-f2a6-480a-ba35-e03a3c9654d0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62631","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.901932354Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.902714539Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.902722995Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.926321573Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.926369827Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.946399551Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.946440248Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.964603986Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.964655596Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.985524304Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:12.985563071Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.005596684Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.005640991Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.024891631Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.024937307Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.045031818Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.045066572Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.065334136Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.065465005Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.083746538Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.083786026Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.102199859Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.10224471Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.121798837Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.121845535Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.140979875Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.14102182Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.158293496Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.15834595Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.176401596Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.176436125Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.193623383Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.193658503Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.211296534Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.211339444Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.22916419Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.22920261Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.247537553Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.247572138Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.265622177Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.265661216Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.283710426Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.283753205Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.284107599Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.284246399Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373653427Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.37368954Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.37369735Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.37370169Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.3737059Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373726235Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373730791Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373735713Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373740982Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373745145Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373750123Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373753451Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.37375708Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373761674Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373764852Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.37376787Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373771039Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373774836Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.373778717Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:13.373833787Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"e74d3cef-aaab-4fea-affe-a54685925451","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized)]","latency_us":470077,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:13.374004257Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"18.077729697s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e74d3cef-aaab-4fea-affe-a54685925451\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:06:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e74d3cef-aaab-4fea-affe-a54685925451\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.502064133Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.502913616Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.502924922Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.523904194Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.523955907Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.540386722Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.540434386Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.56061293Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.560656262Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.578271363Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.578393876Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.595820694Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.595855944Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.616283234Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.616325048Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.635834826Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.635873827Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.656843152Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.65689961Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.67579341Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.675833911Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.694789016Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.694827389Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.714393795Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.714434164Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.732344295Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.732382509Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.749407012Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.749445798Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.768153355Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.768192674Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.786183046Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.786213617Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.803835418Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.803872741Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.823342589Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.82338668Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.841149487Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.841205195Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.859202506Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.859237353Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.877641511Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.877687955Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.878015507Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.878218927Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970024948Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.9701052Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970112897Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970136223Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970147764Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970155779Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.97016568Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970186577Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.97019808Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970209331Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970223178Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970229694Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970238141Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970244845Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970251557Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970259967Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970270438Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970280752Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970295339Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:3001","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:31.970426922Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"8be59e34-2ca7-4cf4-9bae-1dd3af810f87","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:837","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\", TimeNotSynchronized), (\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\", TimeNotSynchronized), (\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\", TimeNotSynchronized), (\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\", TimeNotSynchronized), (\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\", TimeNotSynchronized), (\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\", TimeNotSynchronized), (\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\", TimeNotSynchronized), (\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\", TimeNotSynchronized), (\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\", TimeNotSynchronized), (\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\", TimeNotSynchronized), (\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\", TimeNotSynchronized), (\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\", TimeNotSynchronized), (\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\", TimeNotSynchronized), (\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\", TimeNotSynchronized), (\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\", TimeNotSynchronized), (\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\", TimeNotSynchronized), (\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\", TimeNotSynchronized), (\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\", TimeNotSynchronized), (\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\", TimeNotSynchronized)]","latency_us":466537,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:06:31.970665445Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:465","retry_after":"21.272223322s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8be59e34-2ca7-4cf4-9bae-1dd3af810f87\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:06:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8be59e34-2ca7-4cf4-9bae-1dd3af810f87\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:35.594962126Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35311"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:35.595041019Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"d61444fc-63c2-4258-9b41-62422de0fd2b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35311","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":56,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 40c6fa81-4587-4a16-bb09-2fff99501399, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 7b9123ca-e648-4548-927b-0f1170fc4305, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: baface08-db0f-416e-aebc-5c9cb5f877b0, kind: Service { id: 7b9123ca-e648-4548-927b-0f1170fc4305 }, name: Name(\"ntp-7b9123ca-e648-4548-927b-0f1170fc4305\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 244, 209])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 445a5464-97d8-46cb-9166-d6a1bd785516, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e021f964-253b-4666-b493-d4163110f429, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: c7a10e0d-185c-4d4c-ba53-8bef41552852, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: b7800861-5653-497b-ad56-27d943e37984, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 5ec79911-5195-4752-a161-2b1bf38e251b, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: 70e7e768-b7ab-4c53-bb10-ad562f4df0a9, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 3f5f3991-76a5-4697-905d-55a18f86cab1, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 04a7109c-91dc-4248-84ce-e88adbaf1e1a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External } } } }, OmicronZoneConfig { id: fbae83c1-00ac-4141-8d97-8d932c61ce14, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External } } } }, OmicronZoneConfig { id: 563ec68e-40bf-492a-8838-aed5533b6130, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External } } } }, OmicronZoneConfig { id: f3116b20-bc2b-4361-9725-ec00aa64331c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External } } } }, OmicronZoneConfig { id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External } } } }, OmicronZoneConfig { id: 8daf4293-617d-4f86-9bbf-6d32d9755797, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External } } } }, OmicronZoneConfig { id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External } } } }, OmicronZoneConfig { id: 4b62e2fa-afd7-4315-9043-c99779780ece, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External } } } }, OmicronZoneConfig { id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.301784235Z","hostname":"sock","pid":1618,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:427"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.303029468Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.303044459Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2bb6497-a52a-4f87-9f06-d2763d7b8a3b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.325300893Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.325350019Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 40c6fa81-4587-4a16-bb09-2fff99501399, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.345802012Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.34583954Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c139c1b3-e02d-4faa-8e97-39cd77d3b040, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.365832879Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.365874392Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b837ec5-9f47-45f0-a0ad-0b45cda5746d, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.384859692Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.384908143Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9cda62fc-1c62-4db5-bea1-f6915c6e72b7, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.40333413Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.403367113Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 29c04c78-dae0-44d9-b21b-b3adb1a02fa9, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.420534276Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.420581115Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10f37e08-3cf3-4a6d-9b58-c531ed2b12d9, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.438328366Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.43836433Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3ce5b6ed-8c6a-4176-a669-8e3224034dba, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.456129584Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.456184889Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 445a5464-97d8-46cb-9166-d6a1bd785516, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.473854329Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.473892645Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.491201103Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.491242354Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5ec79911-5195-4752-a161-2b1bf38e251b, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.509726411Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.5097653Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd, dataset_name: DatasetName { pool_name: ZpoolName { id: 3261c18c-ce8f-4256-bb9a-b916c37c65cd, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.527622211Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.52765733Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: fbae83c1-00ac-4141-8d97-8d932c61ce14, dataset_name: DatasetName { pool_name: ZpoolName { id: 3fdd1e8f-902b-4182-b368-f65989a017ec, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.545177132Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.545226042Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 563ec68e-40bf-492a-8838-aed5533b6130, dataset_name: DatasetName { pool_name: ZpoolName { id: 998e9ff7-344d-4c0f-ac09-09fe8899d73a, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.563012657Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.56306798Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f3116b20-bc2b-4361-9725-ec00aa64331c, dataset_name: DatasetName { pool_name: ZpoolName { id: c9f8e23e-40d6-46e7-980e-8af586658d29, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.581184952Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.581229138Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1640b595-cbbb-4e94-a8a9-61579e7cb697, dataset_name: DatasetName { pool_name: ZpoolName { id: ece5c680-a159-4726-9af2-b8b9e799b8a0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.598941462Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.598981175Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8daf4293-617d-4f86-9bbf-6d32d9755797, dataset_name: DatasetName { pool_name: ZpoolName { id: c1ff3dbe-98c6-4a12-9904-ae7e718611dc, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.6170984Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.617133323Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3efa5b01-9e47-47dc-9ff7-86a891f62716, dataset_name: DatasetName { pool_name: ZpoolName { id: 121748b7-d12c-483c-bb9b-e9f8929f1ec8, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.636291455Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.636333494Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 4b62e2fa-afd7-4315-9043-c99779780ece, dataset_name: DatasetName { pool_name: ZpoolName { id: 64f2bb49-5808-4ece-aabc-b25bc06f3536, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.655127074Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.655179931Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18f04ab3-f8b2-4b2e-9d03-bf9933254c9e, dataset_name: DatasetName { pool_name: ZpoolName { id: 1eee6f0c-2a27-4133-94bc-978e69e92ff0, kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.673738848Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.673774886Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.674136487Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.674285863Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.772905972Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.773171266Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.773193438Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.808404432Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.80860598Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.808623333Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.808665437Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.808674859Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.830891797Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.83281976Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.997065662Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:53.997327006Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.011559823Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.011836876Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.011901884Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.024539362Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.02462537Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.047446758Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.047495435Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.04769808Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.053147677Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.057947124Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.062531323Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.067287309Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.071782854Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.07662026Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.081230619Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.201189253Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.205089226Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.209769033Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.214473338Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.218947573Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.223200792Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.227998858Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.232926133Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.238133837Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.243188944Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.248261538Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.25393875Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.259669001Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.265241139Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.271382312Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.277936198Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.283539088Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.289597312Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.295200164Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.300899093Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.358042815Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.398920909Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.431732209Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.463523713Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.495475345Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.497368263Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.497375463Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.563758778Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 145, 5]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.563809245Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.566683715Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.56669099Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.619117979Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 199, 101]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.61916463Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.621761962Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.621776034Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.677097651Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 140, 18]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.677141134Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.678959133Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.678972747Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.732383748Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 138, 219]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.73244691Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.734198537Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.734225152Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.781164091Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 170, 30]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.781206186Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:1277","zone_type":"nexus"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.782094353Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.782108275Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.782202928Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.782208815Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.782527667Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.785031659Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.807970105Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.810543289Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.810580508Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.810586283Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.810698946Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.810706352Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.810798051Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.810804258Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.835893896Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.838283096Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.840281389Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.840299145Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.84034918Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.842801723Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_nexus_e021f964-253b-4666-b493-d4163110f429","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.843070964Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.849685871Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_e021f964-253b-4666-b493-d4163110f429","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.922726619Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.927353124Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.932380483Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.937598153Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.943784976Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.958492498Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.966241795Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:06:54.974745473Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService10\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::17\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_c1ff3dbe-98c6-4a12-9904-ae7e718611dc/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::17\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"9e720c8b-3edb-4589-9382-3b76fd2be17d\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.136524485Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.155931576Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService11\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::10\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::10\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.249278619Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService14\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::14\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_998e9ff7-344d-4c0f-ac09-09fe8899d73a/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::14\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"85e5c3f1-4607-409f-a4d9-3dc0c6612939\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.25749895Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService15\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::f\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::f\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.259963308Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService17\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::19\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_64f2bb49-5808-4ece-aabc-b25bc06f3536/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::19\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"47524317-5246-4c50-ac10-5ab1d02389d4\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.266860818Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.268685111Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.306550462Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.359976895Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62343"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.362022793Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.396352562Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"1ffaf51e-139e-4861-8d11-abdbcabe4005","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:62343","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":7998,"response_code":"200"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.400493588Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService18\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::18\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_121748b7-d12c-483c-bb9b-e9f8929f1ec8/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::18\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"da4d2a4b-e971-406c-85dc-11b7439f2554\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.423070094Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService22\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_1eee6f0c-2a27-4133-94bc-978e69e92ff0/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1a\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"8bca4bc7-7172-4ba9-bc82-2ea1e724e5ab\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.424470656Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService12\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::13\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_3fdd1e8f-902b-4182-b368-f65989a017ec/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::13\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"eb2225b1-10db-4f65-a752-a2f208058176\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.425137502Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService13\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::16\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_ece5c680-a159-4726-9af2-b8b9e799b8a0/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::16\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"7810e984-6433-41c4-85ea-4a775b707880\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.425782287Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.425992812Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.451669389Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.459675396Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.486118491Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService20\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::11\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::11\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.637263619Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.637487094Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService21\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::12\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_3261c18c-ce8f-4256-bb9a-b916c37c65cd/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::12\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"a1b0a35f-671a-48e6-bdc4-4894114fcfe9\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.688784038Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.688988193Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService16\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::15\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_c9f8e23e-40d6-46e7-980e-8af586658d29/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::15\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"4e9b84c9-d28b-4703-a9a6-c190a4f6160a\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.700929321Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:07.701139365Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 69.495108ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:07.976443163Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 41.503533ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:07.990862688Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 65.390293ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.012234116Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 70.98194ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.19460834Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 46.529732ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.429747622Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 62.077322ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.447752712Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 92.618191ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.467416239Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 119.459412ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.484601069Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 105.858738ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.5181744Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 53.371008ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.533986248Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 125.272755ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.552056899Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 131.448252ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.570492528Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 60.144851ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.585482507Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 79.25803ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.602284528Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 37.980052ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.623289767Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 71.714856ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.670211649Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 58.098034ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.710609649Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 73.152331ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.727355939Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 185.045676ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.754290746Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 237.411473ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.774928706Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 251.885469ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.792405243Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 101.789799ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.814233681Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 101.469381ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.830307495Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 235.776196ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.846476701Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 138.48696ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.865416712Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 126.295365ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.878125728Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 91.962563ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.892157435Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService19\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::d\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value=\"b7800861-5653-497b-ad56-27d943e37984\"/>\n        <propval type=\"astring\" name=\"address\" value=\"[fd00:1122:3344:101::d]:12223\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:08.893005504Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 98.262447ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.909323364Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 78.842543ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.922785048Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 60.7209ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.935911683Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 261.11889ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.948877025Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 355.473721ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.960474085Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:08.960631378Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 295.014963ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:08.980135502Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 200.20998ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.00065044Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 169.454817ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.024397452Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 595.379808ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.043674631Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 103.935743ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.058990618Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 375.188097ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.07455509Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 204.007563ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.090690965Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 207.749229ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.106726098Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 413.393674ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.124938422Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 408.385954ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.140634718Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 407.380381ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.180107316Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 348.820226ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.212630723Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 274.079132ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.230363206Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 346.382299ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.243693786Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 433.238435ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.291284694Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 485.955555ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.313599083Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 365.596733ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.330601589Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 806.244387ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.34696993Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 56.486115ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.360413993Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 82.704697ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.433202477Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 529.43864ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.474670362Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 506.470689ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.529826466Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 208.067869ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.545252195Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 637.25931ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.564926683Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 1.108691923s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.582598344Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 807.57344ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.596848765Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 474.183334ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.614610709Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 963.756102ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.628974923Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 508.013579ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.653636677Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 926.958523ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.712998009Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 620.300845ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.74013344Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 269.107701ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.769506874Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 1.145593582s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:09.817684565Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 1.377330995s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.022211636Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 1.350121172s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.057500317Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 1.188028299s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.077015641Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 1.40597297s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.109743993Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 821.851856ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.173135527Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 1.233013374s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.189304954Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 1.055531967s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.219026276Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 1.173465164s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.378898946Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 1.038203319s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.421128449Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 656.153467ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.610541568Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 572.713618ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.6622258Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 736.948096ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.711091798Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService27\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::9\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::9]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.6:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:10.919935869Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:10.920158664Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService26\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::8\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::8]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.5:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:10.936191727Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:10.936469633Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 613.302256ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:10.97670687Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 1.095068717s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.013349743Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 26.75573ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.202967225Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 92.865149ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.261873896Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 53.95042ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.29642132Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 569.197582ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.353864567Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 552.241389ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.450130628Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 751.440659ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.468407703Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 797.702273ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.534304787Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 1.045916432s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.562897812Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 1.258491739s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.581557626Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 1.206301026s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.604365387Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 634.821055ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.626153654Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 121.878784ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.645909994Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 1.436167066s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.668042849Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 166.725754ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.693573994Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 1.338570105s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.715810811Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 978.754957ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.735412247Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 682.566062ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.754526572Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 286.502055ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.786501896Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 365.485499ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.885665354Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 847.116064ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.937189073Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 1.368386455s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:11.985395155Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 370.849227ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.084974843Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 1.408513549s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.119895606Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 675.02739ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.231839631Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 1.167139381s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.262811122Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 634.835087ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.271409122Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 799.885897ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.34394063Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 721.34563ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.449957167Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 659.350746ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.466638936Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 1.080182894s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.624199883Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_e021f964-253b-4666-b493-d4163110f429:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService24\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:12.679806137Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService28\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:12.680231727Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:12.680260858Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:12.689026502Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 705.079907ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.734430579Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 652.246238ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.805517173Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 543.196313ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.832002479Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 978.567401ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.891828162Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 979.107828ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.926084269Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 1.241808394s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:12.941039027Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 70.383724ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.01161135Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 34.380935ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.0223625Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 1.341357883s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.076691114Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 98.665293ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.092546113Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 56.103029ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.2217449Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 572.924427ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.334202657Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 234.814371ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.374158157Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 1.105955363s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.39275806Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 1.398165868s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.408295831Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 890.692593ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.421388986Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 1.05919489s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.436812555Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 595.679294ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.44647423Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 1.024754382s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.459132335Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 142.063553ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.472717901Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 750.714985ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.485478588Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 990.932833ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.498210874Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 1.441549508s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.541094282Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 445.157173ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.624852782Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 386.223005ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.634928286Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 510.031041ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.716485614Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService25\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::c\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:13.7192922Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:13.72145617Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 983.175321ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.889123252Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 895.34021ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.926331998Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 1.182232913s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:13.94746152Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 982.430406ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.041697633Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 1.107465787s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.060577993Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 32.751358ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.072678174Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 1.195124228s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.087584941Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 50.092723ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.121288736Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 209.993852ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.277182184Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 871.891167ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.323354839Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 702.304848ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.337704803Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 544.559412ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.348247208Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 679.850899ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.362877479Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 1.375550118s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.428806043Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 1.406490838s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.49956039Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 937.104146ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.517099848Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 400.942864ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.535282031Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 1.22106635s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.550191078Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 1.469982341s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.564079891Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 1.254037284s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.819599348Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 1.115575645s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.835867267Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 1.457379384s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.887737564Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 515.433442ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.906741946Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 577.296446ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.948736324Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 560.692093ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:14.997867207Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 1.321780181s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.03752273Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 886.991999ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.055491519Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 1.34809338s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.068036351Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 859.84873ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.143435294Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 584.427467ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.182423932Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 762.144244ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.207794222Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 513.661504ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.297695305Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 878.138786ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.434433917Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 577.005831ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.473196389Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 506.004879ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.5389823Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 1.462599191s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.575699715Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService23\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::e\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::e\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"8123\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:2::1\" />\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:15.764196512Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:15.764627652Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 657.60053ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.787961405Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 1.398639018s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.802581306Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 1.420218495s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.822991801Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 605.114872ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.840032137Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 854.701849ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.920035279Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 551.620384ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.956534638Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 663.27121ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.96820283Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 832.863751ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:15.988209256Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 74.73576ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.006598054Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 821.31332ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.021670814Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 1.148436929s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.049087643Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 1.49335553s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.06313554Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 1.22705784s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.076058531Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 1.493617031s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.087956377Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 119.182332ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.10913506Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 108.696396ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.24192429Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 664.711661ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.327753768Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 650.355554ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.359472746Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 1.366609507s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.370751269Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 471.907263ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.385300547Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 988.125336ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.430030798Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 1.004754363s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.460300033Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece\") failed: Property not found. retry in 1.350907535s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.472078623Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 1.31714866s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.523976695Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716\") failed: Property not found. retry in 1.137109376s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.647407207Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 1.455310237s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.788931261Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797\") failed: Property not found. retry in 1.490619856s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.835956756Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1\") failed: Property not found. retry in 770.646879ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.858680354Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 763.505521ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:16.874556364Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9\") failed: Property not found. retry in 1.413385487s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.008225955Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 731.292089ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.02259972Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130\") failed: Property not found. retry in 1.250039968s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.055575277Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 654.656337ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.215165371Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 653.51768ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.230567779Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e\") failed: Property not found. retry in 1.05535872s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.260388323Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 1.149875571s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.320579414Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 822.863116ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.43854772Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 1.266300599s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.480273381Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 936.030414ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.576876319Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 1.145744537s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.604544963Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 1.357303317s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.674304377Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 800.952227ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.760684627Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 626.175655ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.77754881Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 844.910066ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.878696534Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 640.964613ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.899274363Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 1.013241503s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:17.921365197Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 1.245408778s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.264473402Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a\") failed: Property not found. retry in 1.456327277s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.282604384Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 590.669107ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.422799867Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697\") failed: Property not found. retry in 1.318747952s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.491856304Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 693.113993ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.53553647Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 1.458207736s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.56475925Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 807.64392ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.581788317Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 1.111222064s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.752602462Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 1.125318881s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.767847427Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 652.933914ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.784591287Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 1.485659527s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:18.955809852Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 780.583726ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.035333903Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 774.353713ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.050871984Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 1.374369608s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.249539758Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 1.08148874s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.411790484Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 1.014008613s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.460081208Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd\") failed: Property not found. retry in 1.305431144s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.524214341Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c\") failed: Property not found. retry in 1.444016314s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.853990006Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 993.509588ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.868433512Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14\") failed: Property not found. retry in 940.181844ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.882229343Z","hostname":"sock","pid":1618,"zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 1.457366803s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:19.912103349Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984\") failed: Property not found. retry in 1.269925914s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:20.039813881Z","hostname":"sock","pid":1618,"zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c\") failed: Property not found. retry in 1.173209464s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:20.458388302Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 786.155528ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:20.490800047Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 987.349566ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:20.510828063Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 867.26418ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:20.641141346Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 1.178552311s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:20.892591998Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516\") failed: Property not found. retry in 1.048451025s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:21.295748591Z","hostname":"sock","pid":1618,"zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 690.065792ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:21.381861075Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852\") failed: Property not found. retry in 1.016963242s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:21.516329815Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 613.708681ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:21.528688173Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_e021f964-253b-4666-b493-d4163110f429\") failed: Property not found. retry in 1.315878983s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:22.086535716Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 638.477923ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:22.099740073Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 706.359122ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:22.157930387Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 1.319776565s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:22.75399032Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75\") failed: Property not found. retry in 918.071974ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:22.878787134Z","hostname":"sock","pid":1618,"zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b\") failed: Property not found. retry in 1.049235327s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:07:24.085804556Z","hostname":"sock","pid":1618,"zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161801367Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_8daf4293-617d-4f86-9bbf-6d32d9755797"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161888729Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_pantry_3f5f3991-76a5-4697-905d-55a18f86cab1"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161896279Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_nexus_e021f964-253b-4666-b493-d4163110f429"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161905549Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_fbae83c1-00ac-4141-8d97-8d932c61ce14"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.16191881Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_1640b595-cbbb-4e94-a8a9-61579e7cb697"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.16192471Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_563ec68e-40bf-492a-8838-aed5533b6130"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.16193029Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_pantry_70e7e768-b7ab-4c53-bb10-ad562f4df0a9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.16193542Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_nexus_dac1ba5e-0562-40c3-b67a-d11147e0dd75"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.16194059Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_f3116b20-bc2b-4361-9725-ec00aa64331c"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.16194549Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_4b62e2fa-afd7-4315-9043-c99779780ece"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161965091Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_3efa5b01-9e47-47dc-9ff7-86a891f62716"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161970791Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_external_dns_445a5464-97d8-46cb-9166-d6a1bd785516"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161975801Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_oximeter_b7800861-5653-497b-ad56-27d943e37984"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161981481Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_pantry_04a7109c-91dc-4248-84ce-e88adbaf1e1a"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161986641Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_external_dns_b32f647f-5514-4f3a-b22b-09d8ab2bed7c"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161991991Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_4de4be7d-77c9-4097-a1cd-01bf5cbdf7bd"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.161997112Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_nexus_c7a10e0d-185c-4d4c-ba53-8bef41552852"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.162188666Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_crucible_18f04ab3-f8b2-4b2e-9d03-bf9933254c9e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.162201516Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"sled-agent/src/services.rs:2997","zone":"oxz_clickhouse_5ec79911-5195-4752-a161-2b1bf38e251b"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.164871248Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.166730242Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.180104333Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"PUT","req_id":"5ab4ae94-47a2-4245-abe7-91ca5455bee8","remote_addr":"[fd00:1122:3344:101::1]:40623","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":31849710,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.205852372Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1230"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.205969355Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.206104498Z","hostname":"sock","pid":1618,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.214024412Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:677"}
{"msg":"Nexus address: [fd00:1122:3344:101::b]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.216660204Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:685"}
{"msg":"rack_network_config: RackNetworkConfigV1 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.2.174,\n    infra_ip_last: 10.151.2.174,\n    ports: [\n        PortConfigV1 {\n            addresses: [\n                V4(\n                    Ipv4Network {\n                        addr: 10.151.2.174,\n                        prefix: 24,\n                    },\n                ),\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Network {\n                            addr: 0.0.0.0,\n                            prefix: 0,\n                        },\n                    ),\n                    nexthop: 10.151.2.100,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Network {\n        addr: fd00:1122:3344:100::,\n        prefix: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.262939731Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:816"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.269858842Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.419383122Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.585885747Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.787379307Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:25.972915785Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:26.166262414Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:26.492654601Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:26.657013696Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:26.962423544Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:27.204930977Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:27.914623135Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:28.397029322Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:28.904587105Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:30.150918951Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::b]:12221/racks/fffdd475-94d1-496c-86ec-4c108167d272/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:31.246410017Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 400 Bad Request; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"013086a6-b9b0-4efd-ae94-65252dc2e7d2\", \"content-length\": \"123\", \"date\": \"Sat, 30 Mar 2024 04:07:32 GMT\"}; value: Error { error_code: Some(\"InvalidRequest\"), message: \"rack not found\", request_id: \"013086a6-b9b0-4efd-ae94-65252dc2e7d2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:32.710722438Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8b205a48-bdc6-4bb7-9290-406d533c05b3\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8b205a48-bdc6-4bb7-9290-406d533c05b3\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:34.390628764Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e99d58bc-9eaa-41c5-9c3d-f2099892cfc9\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e99d58bc-9eaa-41c5-9c3d-f2099892cfc9\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:36.901726196Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:37.46476924Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44444"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:37.465781663Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"4ed3b524-6308-422f-bcbf-b957b3b1a875","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44444","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":224,"response_code":"200"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3e76363b-74f9-4fe6-ada6-4b9bc8145c88\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3e76363b-74f9-4fe6-ada6-4b9bc8145c88\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:38.517397468Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7a9016ae-6a13-456b-8a2d-32b06c216300\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7a9016ae-6a13-456b-8a2d-32b06c216300\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:41.009612931Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0754571e-a0f5-4b16-b9ab-f8b539b57e99\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0754571e-a0f5-4b16-b9ab-f8b539b57e99\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:45.103370746Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c78e6197-ebaa-4f46-9b6a-2a6cde974f6e\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c78e6197-ebaa-4f46-9b6a-2a6cde974f6e\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:48.922666435Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"97f3709d-72a5-4d60-b9f3-877f52561c53\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"97f3709d-72a5-4d60-b9f3-877f52561c53\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:52.383984101Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Failed to handoff to nexus: Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05d2e9a7-00e4-40dc-9b50-ebc42e192bc1\", \"content-length\": \"124\", \"date\": \"Sat, 30 Mar 2024 04:07:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05d2e9a7-00e4-40dc-9b50-ebc42e192bc1\" }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:07:59.337537376Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:873"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:04.005756332Z","hostname":"sock","pid":1618,"component":"NexusNotifierTask","sled_id":"82d35b43-2f1a-463b-bc45-db2841fe70e1","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:07.231909592Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61843"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:07.234917282Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"dc4aac72-321d-4260-9a55-0ac1a53e59a7","remote_addr":"[fd00:1122:3344:101::b]:61843","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2968,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:07.632439143Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41746"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:07.632527415Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"3edb7f98-30de-42d7-abaa-7c18436cb18a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41746","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":67,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.699789463Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:62888"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.706323935Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3bbcdf6d-e84e-46e6-b81a-7352cfe2d662, kind: Service { id: 445a5464-97d8-46cb-9166-d6a1bd785516 }, name: Name(\"external-dns-445a5464-97d8-46cb-9166-d6a1bd785516\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 140, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e9e942d9-53e0-4893-bd89-6f493ba577be, kind: Service { id: b32f647f-5514-4f3a-b22b-09d8ab2bed7c }, name: Name(\"external-dns-b32f647f-5514-4f3a-b22b-09d8ab2bed7c\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 219])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1c502ff1-b12b-46a4-a410-31cf1417cf3e, kind: Service { id: e021f964-253b-4666-b493-d4163110f429 }, name: Name(\"nexus-e021f964-253b-4666-b493-d4163110f429\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 145, 5])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a2cf1046-7e91-4a6f-a668-88a9ee57df93, kind: Service { id: c7a10e0d-185c-4d4c-ba53-8bef41552852 }, name: Name(\"nexus-c7a10e0d-185c-4d4c-ba53-8bef41552852\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 170, 30])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: d40889b8-707c-4a66-8721-091298fe0458, kind: Service { id: dac1ba5e-0562-40c3-b67a-d11147e0dd75 }, name: Name(\"nexus-dac1ba5e-0562-40c3-b67a-d11147e0dd75\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 101])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.707468852Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.707502352Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.707508973Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.707526453Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.707995624Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.708009784Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.708037785Z","hostname":"sock","pid":1618,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9a7710e6-b332-4967-9885-46254389dbee","remote_addr":"[fd00:1122:3344:101::b]:62888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":8213,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.709317125Z","hostname":"sock","pid":1618,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:883"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.993209072Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:37913"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:08.995163557Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:09.08079422Z","hostname":"sock","pid":1618,"uri":"/inventory","method":"GET","req_id":"a655dbee-d670-43ed-a860-e91587e33b14","remote_addr":"[fd00:1122:3344:101::b]:37913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87514,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:09.081777813Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:09.082127091Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:09.083873462Z","hostname":"sock","pid":1618,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:09.084996808Z","hostname":"sock","pid":1618,"uri":"/omicron-zones","method":"GET","req_id":"ed5a75bc-de57-44e9-8b6b-8ae72ec58850","remote_addr":"[fd00:1122:3344:101::b]:37913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3921,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:32.479562277Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::d]:61235"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:32.491227388Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"e74d2de2-5aab-4a51-9b8a-5d7905a4e065","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":11644,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:32.978419307Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59564"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.064612582Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c4a6b8c3-2577-46c2-a8d4-7336a86e382c","remote_addr":"[fd00:1122:3344:101::b]:59564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86139,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.064755436Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:42523"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.118079787Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"832801b2-204b-4eea-8f95-4e0e09e2eef2","remote_addr":"[fd00:1122:3344:101::c]:42523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53244,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.118355003Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:64679"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.122564531Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"e9d3380b-db8a-4ee6-b360-6646de159fcc","remote_addr":"[fd00:1122:3344:101::b]:59564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4192,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.124772653Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"a7c2d0df-7cc8-44eb-81ac-eb0aa659c6b4","remote_addr":"[fd00:1122:3344:101::c]:42523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":4200,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.154882273Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.154943374Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.155768033Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.156764076Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.156789077Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"PUT","req_id":"eb921583-aae8-4dfe-be7b-ef7628fa4cdf","remote_addr":"[fd00:1122:3344:101::c]:42523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3152,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.167523077Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"97323cd8-2c42-4325-b79f-31318af5a5a7","remote_addr":"[fd00:1122:3344:101::a]:64679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48886,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.167800323Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c59055c5-4bd3-4eb5-95c4-977dd672df3e","remote_addr":"[fd00:1122:3344:101::a]:64679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.203205637Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.203233338Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.203437243Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.203608416Z","hostname":"sock","pid":1618,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:33.203618007Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"PUT","req_id":"7c4d8b78-a13d-4109-a047-a3b9e5bf5e4e","remote_addr":"[fd00:1122:3344:101::b]:59564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":523,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:37.66569662Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57855"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:37.666369525Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"2d1c6447-499a-4bef-9950-35b895e1c61a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57855","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":616,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:08:59.058620055Z","hostname":"sock","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.480659334Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"8f6248c7-529d-4c6a-9a2c-fb4e38712a8d","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":535,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.669787075Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:55588"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.715685033Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"25af9b01-4e4b-41a1-bd9e-f4994f6381ee","remote_addr":"[fd00:1122:3344:101::c]:55588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45844,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.716083613Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"4b8d0b10-efc0-49ed-9f5a-0313dc341546","remote_addr":"[fd00:1122:3344:101::c]:55588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":149,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.723659649Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:59547"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.766980737Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"0477d362-3288-47e3-9db4-172b1361d53a","remote_addr":"[fd00:1122:3344:101::b]:59547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43217,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.767161201Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:62777"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.771204616Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c633c6ee-ea5f-43c0-8928-c7b0bfbeee8a","remote_addr":"[fd00:1122:3344:101::b]:59547","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3823,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.808959114Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"e79e5cc9-7a47-4b2a-bacc-72eea29d89c2","remote_addr":"[fd00:1122:3344:101::a]:62777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41678,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:02.809302892Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"e47a7654-6d65-4a00-a86a-f4cc51bca823","remote_addr":"[fd00:1122:3344:101::a]:62777","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":153,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:07.697771008Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47195"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:07.69870459Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"62a01627-c4d8-49d8-8f51-59ae0930aaba","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47195","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":902,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:32.480317567Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"c1ce7cdf-55cc-4ef4-866c-ffe3cce541f0","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":508,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:32.953435268Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:43406"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.001549937Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"5d4a0b80-42df-445d-96cc-ca7028536f60","remote_addr":"[fd00:1122:3344:101::b]:43406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48063,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.001725212Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:60754"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.005275444Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"58889a32-5cb1-4aa1-86e5-a250aac77c91","remote_addr":"[fd00:1122:3344:101::b]:43406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3435,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.046238437Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"71fef993-5b5b-4adb-bf39-bfa32d72d941","remote_addr":"[fd00:1122:3344:101::a]:60754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":44432,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.046622836Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"0abc5659-c25e-474a-bb89-a140c159d5b1","remote_addr":"[fd00:1122:3344:101::a]:60754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":127,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.055877092Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:34413"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.099518617Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"830195a1-ba4a-4321-bf44-d6a413430e66","remote_addr":"[fd00:1122:3344:101::c]:34413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43572,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:33.099786344Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"02d39f6a-30f7-45e1-b3bb-0c2d74088a5d","remote_addr":"[fd00:1122:3344:101::c]:34413","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":90,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:37.732120209Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43823"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:09:37.732307783Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"290002c8-9d14-4a7d-8ec5-4450e3bad45b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43823","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":138,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.480419901Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"b3b66a66-1e3b-4645-b340-d64c3eff9ce2","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":617,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.653275754Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:46853"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.701860735Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"6197a9e0-f8ad-468a-8dd5-a4ce6ad565bd","remote_addr":"[fd00:1122:3344:101::b]:46853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48516,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.702154741Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"368f3546-6c80-450f-be74-c5efcce7caa6","remote_addr":"[fd00:1122:3344:101::b]:46853","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.737535385Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:44107"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.786807502Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"ec89d5bc-7d07-4118-b842-03184070c684","remote_addr":"[fd00:1122:3344:101::c]:44107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49167,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.786985356Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:42933"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.790557969Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"b9a6b588-1ba0-46f7-870b-b74f98cc0831","remote_addr":"[fd00:1122:3344:101::c]:44107","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3431,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.834070282Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"ba3df2b2-779f-49ed-b818-762952e880ad","remote_addr":"[fd00:1122:3344:101::a]:42933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47001,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:02.834456921Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"2a3819dc-287c-4fb3-86fd-ad7ebfa49e46","remote_addr":"[fd00:1122:3344:101::a]:42933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":196,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:07.763743678Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33202"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:07.763915872Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"e2c900fe-93e2-45a5-a98a-e25c8b5dea41","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33202","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":69,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:32.480091155Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"0ceefddd-d3b7-4ec5-9ce2-bd93e1ddcdfe","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":610,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:32.811099108Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:50366"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:32.859706379Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"59df6f7a-9bbf-434b-a6bb-c4d4b72880b4","remote_addr":"[fd00:1122:3344:101::c]:50366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48573,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:32.859969665Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"1f3f77a4-aefd-4122-9e15-bf040bdae9df","remote_addr":"[fd00:1122:3344:101::c]:50366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":101,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:32.881562318Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:45300"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:32.929093034Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"beafc8d4-31cf-491a-b4a7-8a8bff3bddc1","remote_addr":"[fd00:1122:3344:101::b]:45300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47441,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:32.929365871Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"2fffd909-4e12-45bc-ac8e-6fa8c9b8004c","remote_addr":"[fd00:1122:3344:101::b]:45300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":110,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:33.130073121Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:53523"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:33.178077208Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"3a51b87a-2acd-4ddf-bc36-acc2a9c7e6b1","remote_addr":"[fd00:1122:3344:101::a]:53523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47903,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:33.178326674Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"521b8a85-76fe-424c-8ee5-301e1467df50","remote_addr":"[fd00:1122:3344:101::a]:53523","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":95,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:37.7952053Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54234"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:10:37.795314113Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"a96c3c25-6b66-4dbf-8041-be26220589e7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54234","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":102,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.479309979Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"bc041ab2-a970-44eb-aebb-d353c4737903","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":572,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.662698797Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:47858"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.710251684Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c339bf2f-3243-4d5d-8271-b40d0c49a86c","remote_addr":"[fd00:1122:3344:101::b]:47858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47483,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.71054794Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"428ebfac-2699-4f36-b776-9259540b87d0","remote_addr":"[fd00:1122:3344:101::b]:47858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":113,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.751989505Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:51034"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.793524431Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"09698e87-06bf-41fd-9429-fe59057018ac","remote_addr":"[fd00:1122:3344:101::c]:51034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":41449,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.793779728Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"5f9f8c0e-bc67-4661-94d0-25168e0c5fbd","remote_addr":"[fd00:1122:3344:101::c]:51034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.831890704Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:55711"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.878691704Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"bc17d8ee-e615-4599-9450-e87f6dd1e59b","remote_addr":"[fd00:1122:3344:101::a]:55711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46714,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:02.878914249Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"c7eb0494-c572-43dc-9b3a-07f328616ea3","remote_addr":"[fd00:1122:3344:101::a]:55711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":98,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:07.826209225Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42374"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:07.826327668Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"4a60930f-a496-4dd7-bb46-7e80ef4fb023","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42374","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":108,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:32.479094006Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"2d9b3d9b-7f32-4766-aef0-b7d4b5595db7","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":598,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:32.915409591Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:50489"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:32.964533484Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"520cd66e-d2b3-4c83-b17b-c65242427e26","remote_addr":"[fd00:1122:3344:101::b]:50489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":49071,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:32.96479491Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"43159cff-60c9-4859-b2a4-eade7f578a18","remote_addr":"[fd00:1122:3344:101::b]:50489","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:32.978617092Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:48681"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:33.026696081Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"5da22774-3c67-4d62-a005-6178207278bc","remote_addr":"[fd00:1122:3344:101::c]:48681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47973,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:33.026950877Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54247"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:33.030980791Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"544dbbbf-e30d-4d67-945e-9b2daea02c8c","remote_addr":"[fd00:1122:3344:101::c]:48681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3903,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:33.07392774Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"154b05b1-6da1-40ff-adf3-435c7f8975b6","remote_addr":"[fd00:1122:3344:101::a]:54247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46849,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:33.074263148Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"d694707d-8cca-44ae-9e58-95a1f420342b","remote_addr":"[fd00:1122:3344:101::a]:54247","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":150,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:37.857680859Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54198"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:11:37.857794002Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"852571eb-5e12-45fa-bcee-0e30c57438de","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:54198","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.478398182Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"761028d2-8a55-42e2-afc9-a8a2d4f64faa","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":532,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.747477235Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:43243"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.795131204Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"184635b3-5c47-4b3f-b190-8a17c7669aaf","remote_addr":"[fd00:1122:3344:101::c]:43243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47602,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.795329268Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:35515"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.798651015Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"d2db0a1d-a3fc-47f6-b7a4-e153ebd89af3","remote_addr":"[fd00:1122:3344:101::c]:43243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3192,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.842197179Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"50f3a1c7-e4fa-4eb6-8ae0-7dfb430d609d","remote_addr":"[fd00:1122:3344:101::b]:35515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46772,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.842546607Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"1ff60188-675f-43bc-946d-29ce5816b8ca","remote_addr":"[fd00:1122:3344:101::b]:35515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":141,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.896097083Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39144"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.939245208Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"d9389166-da2c-437f-b8f1-eae86952f8db","remote_addr":"[fd00:1122:3344:101::a]:39144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43084,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:02.939905663Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"e7b46c89-aeef-428a-9b30-2a38cfddc7d9","remote_addr":"[fd00:1122:3344:101::a]:39144","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":119,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:07.890108385Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47088"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:07.890226938Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"a55be26e-244c-4bc7-b228-fc912d6b58a8","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47088","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"}
{"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:08.857610051Z","hostname":"sock","pid":1618,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:08.857699693Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:08.896373473Z","hostname":"sock","pid":1618,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999993s","next_cleanup":"Instant { tv_sec: 3331, tv_nsec: 949755761 }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:32.479400839Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"bbce96d8-8f50-41ef-8378-6cf74545830c","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1109,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:32.908056615Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:61990"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:32.961694793Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"8e13057d-9e63-432c-9b17-169aa2c9e0e3","remote_addr":"[fd00:1122:3344:101::b]:61990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53585,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:32.962034411Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"56479585-be70-4f7b-9791-f35efec0de8a","remote_addr":"[fd00:1122:3344:101::b]:61990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":109,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:32.967707213Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:60652"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:33.021139337Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"41349909-d921-41ea-952a-236373814d30","remote_addr":"[fd00:1122:3344:101::c]:60652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":53304,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:33.021669259Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"f61d5a96-bae3-4b01-b51a-1794f49d9e6d","remote_addr":"[fd00:1122:3344:101::c]:60652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":225,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:33.065754455Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:49726"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:33.116634329Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"e8aa4923-957f-44bf-93ee-af957713c4b0","remote_addr":"[fd00:1122:3344:101::a]:49726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":50846,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:33.116904215Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"a8e71518-3aa7-481c-85e6-8c7d63603a23","remote_addr":"[fd00:1122:3344:101::a]:49726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:37.923726146Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36180"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:12:37.923828559Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"1282ea37-4948-490e-b35b-7b5354d86d9e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36180","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":86,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.477238493Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"cb707916-213d-470a-b931-ee255a794f35","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":532,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.738474403Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:39508"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.787454223Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"dfc20b46-2a04-41eb-a7f8-85c93ec0fb16","remote_addr":"[fd00:1122:3344:101::b]:39508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":48939,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.787637557Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:36642"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.78774977Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:55622"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.791284282Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"5119edeb-2629-4e19-96ae-26c459f42a78","remote_addr":"[fd00:1122:3344:101::b]:39508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":3397,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.83116045Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"8e43440f-00a4-4358-838e-0d7bca9fb2be","remote_addr":"[fd00:1122:3344:101::a]:36642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43403,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.834667392Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"67ea54dc-83fc-4e31-b3ba-7ff7cc11dc93","remote_addr":"[fd00:1122:3344:101::a]:36642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":632,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.875286657Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"3704a7f5-bac5-456d-a07d-c914ea3b42ae","remote_addr":"[fd00:1122:3344:101::c]:55622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":87348,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:02.875657776Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"32cbba6f-e50f-4335-a331-fe3f82cfeede","remote_addr":"[fd00:1122:3344:101::c]:55622","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":139,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:07.955955752Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34368"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:07.956072035Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"49d60032-6bd7-45f9-a0a4-5b1ba3a91fa5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34368","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":106,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:32.475842997Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"979a14a2-462b-4073-babf-5717c08eedcb","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":687,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:32.877234048Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:39614"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:32.923887084Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"9e56386d-53ae-41dc-adfd-ac910df951a9","remote_addr":"[fd00:1122:3344:101::c]:39614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46603,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:32.924174101Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"7ceee9e2-3b78-4184-a971-c3ca1c6162a4","remote_addr":"[fd00:1122:3344:101::c]:39614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":104,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:32.945982339Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:34108"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:32.993272939Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"82407321-49df-47a5-a1ac-f092bd1de2d2","remote_addr":"[fd00:1122:3344:101::b]:34108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":47181,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:32.994142869Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"70970c7c-dc63-4d3d-990f-f65715603dc5","remote_addr":"[fd00:1122:3344:101::b]:34108","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":677,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:33.047730507Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:41652"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:33.0916974Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"a77181e7-de01-4243-bad7-083994c1fd5f","remote_addr":"[fd00:1122:3344:101::a]:41652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":43877,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:33.091942126Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"e42625c3-79c4-4714-bf81-d676f77296ce","remote_addr":"[fd00:1122:3344:101::a]:41652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":91,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:37.987181266Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40424"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:37.987299379Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"af53eae4-7862-464f-85d2-44f7ade3fa4d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:40424","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":92,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:13:59.058678293Z","hostname":"sock","pid":1618,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:801"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.474691715Z","hostname":"sock","pid":1618,"uri":"/metrics/collect/82d35b43-2f1a-463b-bc45-db2841fe70e1","method":"GET","req_id":"a5bc9e3b-602c-4200-aa0a-7b9fce94297b","remote_addr":"[fd00:1122:3344:101::d]:61235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":1434,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.616130177Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::b]:64057"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.668187638Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"c332e6fd-2f30-436e-9918-aaaf9bba9cf6","remote_addr":"[fd00:1122:3344:101::b]:64057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":52007,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.668513526Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"360e4e5e-0965-4869-9ae8-aa6c842616e5","remote_addr":"[fd00:1122:3344:101::b]:64057","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":107,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.672226522Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47756"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.718082579Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"7541d918-0be2-47ac-91ba-2428f3903b34","remote_addr":"[fd00:1122:3344:101::c]:47756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":45749,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.718356586Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"9df43c6d-2bb1-4a3f-ab72-86eecd84e864","remote_addr":"[fd00:1122:3344:101::c]:47756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":93,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.738256949Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:54588"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.7851445Z","hostname":"sock","pid":1618,"uri":"/switch-ports","method":"POST","req_id":"869bda19-4c69-4696-a76f-59fa36957454","remote_addr":"[fd00:1122:3344:101::a]:54588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":46796,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:02.785478088Z","hostname":"sock","pid":1618,"uri":"/network-bootstore-config","method":"GET","req_id":"e1c3ebaf-9b9e-487f-917f-5e96019497bf","remote_addr":"[fd00:1122:3344:101::a]:54588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":129,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.421363909Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39657"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.432695533Z","hostname":"sock","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:521","metadata":"InstanceMetadata { silo_id: 526ce5fd-736c-4507-9f2c-92616164421e, project_id: 0e8b9694-053d-48ae-95a5-962ff2daf5d0 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-30T04:14:04.032107Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(8183111))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-d6a148c02c4\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 11cbf034-866c-4335-91ad-b535bf1549f3, read_only_parent: Some(Volume { block_size: 512, id: 847a20bb-3def-4685-8e01-021b3c64c5ed, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9eb53ce0-8b89-446e-9579-8349cabe095c, key: Some(\"84Xq92PoMgux1W4mK2zCW+UeY5K6q0SRllUdvLOmLXw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 11cbf034-866c-4335-91ad-b535bf1549f3, key: Some(\"AmDKpsJJohbF1V4UzJFCNaPqBs1usBe7u86hQ0kNDHk=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAlwiF+WH5YAADCIX5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAJsIhflh+WAAAwiF+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwYjllNmZiZS1mZjQ3LTQ4ZmQtYmY3MS1kMTJmNmVlZWEwYTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTnU0R09RSnFsZFllQ29nbHJ5STNhTmJrWFlQaDZiaU9iQWxFTUNHZDJFeiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.432778875Z","hostname":"sock","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:555","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.433574383Z","hostname":"sock","pid":1618,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(8183111))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b1cfb633-738e-49c6-9360-e3e68b6f58af, kind: Instance { id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 248, 15, 91])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(8183111), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-d6a148c02c4\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 11cbf034-866c-4335-91ad-b535bf1549f3, read_only_parent: Some(Volume { block_size: 512, id: 847a20bb-3def-4685-8e01-021b3c64c5ed, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9eb53ce0-8b89-446e-9579-8349cabe095c, key: Some(\"84Xq92PoMgux1W4mK2zCW+UeY5K6q0SRllUdvLOmLXw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 11cbf034-866c-4335-91ad-b535bf1549f3, key: Some(\"AmDKpsJJohbF1V4UzJFCNaPqBs1usBe7u86hQ0kNDHk=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAlwiF+WH5YAADCIX5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAJsIhflh+WAAAwiF+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwYjllNmZiZS1mZjQ3LTQ4ZmQtYmY3MS1kMTJmNmVlZWEwYTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTnU0R09RSnFsZFllQ29nbHJ5STNhTmJrWFlQaDZiaU9iQWxFTUNHZDJFeiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-03-30T04:14:04.032107Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.435660522Z","hostname":"sock","pid":1618,"uri":"/instances/0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","method":"PUT","req_id":"db548fc2-3a42-4c99-9186-0ef15c740946","remote_addr":"[fd00:1122:3344:101::a]:39657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":14254,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.526112697Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:33442"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.528271497Z","hostname":"sock","pid":1618,"uri":"/instances/0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8/state","method":"PUT","req_id":"ed7036e4-8405-4fec-9708-ff35e01a079d","remote_addr":"[fd00:1122:3344:101::a]:33442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":2080,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.55418619Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 248, 15, 91]), slot: 0, vni: Vni { inner: 8183111 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.554246452Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.587570677Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:04.609226402Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlInstance0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1:0\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"12400\"/>\n        <propval type=\"astring\" name=\"metric_addr\" value=\"[fd00:1122:3344:101::a]:12221\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:06.62094737Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:06.621175186Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 61.982882ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:06.823470454Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 93.067545ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:06.904442468Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 236.207992ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:07.010611419Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 504.993403ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:07.260161677Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 783.288496ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:07.775569762Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:08.018559367Z","hostname":"sock","pid":1618,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57351"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:08.019244083Z","hostname":"sock","pid":1618,"uri":"/baseboard","method":"GET","req_id":"09521bc2-8326-482f-a440-e60a358fad1b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57351","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":678,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 887.932324ms","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:08.568632339Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 1.064213294s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:09.465278917Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\") failed: Property not found. retry in 1.49259339s","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:10.539628761Z","hostname":"sock","pid":1618,"zone":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.060405554Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.069712161Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.113814067Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"}
{"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAlwiF+WH5YAADCIX5YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAJsIhflh+WAAAwiF+WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwYjllNmZiZS1mZjQ3LTQ4ZmQtYmY3MS1kMTJmNmVlZWEwYTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTnU0R09RSnFsZFllQ29nbHJ5STNhTmJrWFlQaDZiaU9iQWxFTUNHZDJFeiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-d6a148c02c4\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 11cbf034-866c-4335-91ad-b535bf1549f3, read_only_parent: Some(Volume { block_size: 512, id: 847a20bb-3def-4685-8e01-021b3c64c5ed, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 9eb53ce0-8b89-446e-9579-8349cabe095c, key: Some(\"84Xq92PoMgux1W4mK2zCW+UeY5K6q0SRllUdvLOmLXw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::14]:19001\", \"[fd00:1122:3344:101::13]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 11cbf034-866c-4335-91ad-b535bf1549f3, key: Some(\"AmDKpsJJohbF1V4UzJFCNaPqBs1usBe7u86hQ0kNDHk=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 0e8b9694-053d-48ae-95a5-962ff2daf5d0, silo_id: 526ce5fd-736c-4507-9f2c-92616164421e }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.119197462Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.205753176Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-03-30T04:14:12.206313169Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.206470973Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.206489393Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-30T04:14:12.206313169Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }","propolis_id":"cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.206500193Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }, propolis_id: cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-03-30T04:14:12.206313169Z } }","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-03-30T04:14:12.261141915Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.261165066Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.261191466Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-30T04:14:12.261141915Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }","propolis_id":"cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:12.261207067Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }, propolis_id: cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-03-30T04:14:12.261141915Z } }","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.923845714Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::a]:39471"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.924463708Z","hostname":"sock","pid":1618,"uri":"/instances/0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8/state","method":"PUT","req_id":"e6784c27-6f24-4f70-82f7-bd5200aa3cb9","remote_addr":"[fd00:1122:3344:101::a]:39471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":546,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-03-30T04:14:21.924568191Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.924592521Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.924611902Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-30T04:14:21.924568191Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }","propolis_id":"cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.924632232Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-03-30T04:14:04.060809389Z }, propolis_id: cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-03-30T04:14:21.924568191Z } }","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-03-30T04:14:21.949408259Z }","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.949433929Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.94946961Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-03-30T04:14:21.949408259Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-30T04:14:21.949408259Z }","propolis_id":"cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.949479841Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.949504721Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.949664445Z","hostname":"sock","pid":1618,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:21.949707706Z","hostname":"sock","pid":1618,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/4d567028-d7e0-4e3c-9e3e-6a96da036239/debug/bundle/zone\", \"/pool/int/c653edc4-5059-41c6-afb0-ff8ae7c3fb4a/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/3261c18c-ce8f-4256-bb9a-b916c37c65cd/crypt/debug\", \"/pool/ext/3fdd1e8f-902b-4182-b368-f65989a017ec/crypt/debug\", \"/pool/ext/998e9ff7-344d-4c0f-ac09-09fe8899d73a/crypt/debug\", \"/pool/ext/c9f8e23e-40d6-46e7-980e-8af586658d29/crypt/debug\", \"/pool/ext/ece5c680-a159-4726-9af2-b8b9e799b8a0/crypt/debug\", \"/pool/ext/c1ff3dbe-98c6-4a12-9904-ae7e718611dc/crypt/debug\", \"/pool/ext/121748b7-d12c-483c-bb9b-e9f8929f1ec8/crypt/debug\", \"/pool/ext/64f2bb49-5808-4ece-aabc-b25bc06f3536/crypt/debug\", \"/pool/ext/1eee6f0c-2a27-4133-94bc-978e69e92ff0/crypt/debug\"] }","zone_name":"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:22.546102615Z","hostname":"sock","pid":1618,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1236","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82\", bundle_id: f31dcd3f-5131-4adb-b3f7-c33f9360ecef }, time_created: 2024-03-30T04:14:21.950692979Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:22.546198468Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:24.307534439Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:24.334426565Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-30T04:14:21.949408259Z }, propolis_id: cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-30T04:14:24.334360953Z } }","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:24.417385956Z","hostname":"sock","pid":1618,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:765","remote_addr":"[fd00:1122:3344:101::c]:47786"}
{"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-03-30T04:14:24.418171994Z","hostname":"sock","pid":1618,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:24.418187355Z","hostname":"sock","pid":1618,"uri":"/v2p/b1cfb633-738e-49c6-9360-e3e68b6f58af","method":"DELETE","req_id":"6c0bd8c5-4c33-43a5-a92b-6d55a4b21470","remote_addr":"[fd00:1122:3344:101::c]:47786","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/29ae98d/dropshot/src/server.rs:849","latency_us":736,"response_code":"204"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:24.648367832Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-03-30T04:14:21.949408259Z }, propolis_id: cd55bd3d-6563-4ba0-9af2-3a68ae4e5a82, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-03-30T04:14:24.334360953Z } }","instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-03-30T04:14:24.651067764Z","hostname":"sock","pid":1618,"instance_id":"0b9e6fbe-ff47-48fd-bf71-d12f6eeea0a8","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}