[ May 15 07:08:05 Enabled. ]
[ May 15 07:08:05 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ]
[ May 15 07:08:05 Method "start" exited with status 0. ]
[ May 15 07:08:05 Rereading configuration. ]
[ May 15 07:08:05 No 'refresh' method defined.  Treating as :true. ]
note: configured to log to "/dev/stdout"
{"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:05.477374035Z","hostname":"sock","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"}
{"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:05.477542608Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"}
{"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:05.530124322Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"}
{"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:05.530168523Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"}
{"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:05.641205023Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"}
{"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:05.650033577Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"}
{"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:05.65836018Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.258924284Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"}
{"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.397504487Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"}
{"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.397542038Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"}
{"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.397669091Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"}
{"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.397766803Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"}
{"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.397776783Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"}
{"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.397800194Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"}
{"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.400895108Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"}
{"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:06.400901368Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"}
{"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.400924019Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"}
{"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.400930679Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.400944849Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.4009924Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.401054581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.401066511Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.401096652Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.413162183Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.413199174Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.514050462Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal }"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.714065943Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.714100604Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.714105924Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.714111014Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.714121795Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.714125985Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.714144335Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.722187012Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.722211863Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:06.806947656Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }","zpool":"ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003717089Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00377748Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00378331Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00379547Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003811231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003815381Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003819961Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003824861Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003837381Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003842541Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003847621Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003859742Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003906113Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003914983Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003935353Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003940223Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003963224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.003967284Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004010655Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004025665Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004040915Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004045695Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004049786Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004053666Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004056906Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004083146Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004105497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004109797Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004113847Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004117557Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004120787Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004132077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004135587Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004150408Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004154918Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004158638Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004162418Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004174638Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004198669Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004202289Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004206909Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004210519Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004214099Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004220989Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004236259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004240059Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00424632Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00424948Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00425354Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"}
{"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00426012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.0042698Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.00427349Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004291951Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004296471Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004299611Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"}
{"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004309131Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004312221Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004338692Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004343692Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004347542Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004371142Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004437074Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004453334Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004462964Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004480144Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004484775Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"}
{"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004488665Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"}
{"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004517325Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"}
{"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.004522085Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"}
{"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.015698238Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:711"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.015729449Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999886978s","next_cleanup":"Instant { tv_sec: 1251, tv_nsec: 643535330 }"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.01577848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.01579856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.015866301Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.015880352Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.018894524Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"}
{"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.018970396Z","hostname":"sock","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Injecting RSS configuration: RackInitializeRequest {\n    trust_quorum_peers: None,\n    bootstrap_discovery: OnlyOurs,\n    ntp_servers: [\n        \"0.pool.ntp.org\",\n    ],\n    dns_servers: [\n        1.1.1.1,\n        9.9.9.9,\n    ],\n    internal_services_ip_pool_ranges: [\n        V4(\n            Ipv4Range {\n                first: 10.151.2.164,\n                last: 10.151.2.173,\n            },\n        ),\n    ],\n    external_dns_ips: [\n        10.151.2.164,\n        10.151.2.165,\n    ],\n    external_dns_zone_name: \"oxide.test\",\n    external_certificates: \"<redacted>\",\n    recovery_silo: RecoverySiloConfig {\n        silo_name: Name(\n            \"recovery\",\n        ),\n        user_name: UserId(\n            \"recovery\",\n        ),\n        user_password_hash: NewPasswordHash(\n            PasswordHashString {\n                string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n                encoding: B64,\n            },\n        ),\n    },\n    rack_network_config: RackNetworkConfigV1 {\n        rack_subnet: Ipv6Network {\n            addr: fd00:1122:3344:100::,\n            prefix: 56,\n        },\n        infra_ip_first: 10.151.2.174,\n        infra_ip_last: 10.151.2.174,\n        ports: [\n            PortConfigV1 {\n                routes: [\n                    RouteConfig {\n                        destination: V4(\n                            Ipv4Network {\n                                addr: 0.0.0.0,\n                                prefix: 0,\n                            },\n                        ),\n                        nexthop: 10.151.2.100,\n                        vlan_id: None,\n                    },\n                ],\n                addresses: [\n                    V4(\n                        Ipv4Network {\n                            addr: 10.151.2.174,\n                            prefix: 24,\n                        },\n                    ),\n                ],\n                switch: Switch0,\n                port: \"qsfp0\",\n                uplink_port_speed: Speed40G,\n                uplink_port_fec: None,\n                bgp_peers: [],\n                autoneg: false,\n            },\n        ],\n        bgp: [],\n        bfd: [],\n    },\n    allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019246032Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1003"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019260252Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019309573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019361784Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019377374Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019391305Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1063"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019423566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019433036Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019441766Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019460376Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1104"}
{"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019464696Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:118"}
{"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019468476Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:91"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019477267Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019502897Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019829264Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019964867Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:138"}
{"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.019970707Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1137"}
{"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020007698Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020014628Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020184281Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020307554Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020344385Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: acb2a796-8e9f-4811-b601-726a12d79b0b, rack_id: 89a4c48a-3dff-478b-98cb-9f441a913804, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }"}
{"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020546899Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:53471","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"}
{"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: acb2a796-8e9f-4811-b601-726a12d79b0b, rack_id: 89a4c48a-3dff-478b-98cb-9f441a913804, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net(Ipv6Network { addr: fd00:1122:3344:101::, prefix: 64 }) } } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020634891Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"}
{"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020644911Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"}
{"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020670341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020675341Z","hostname":"sock","pid":1555,"file":"sled-agent/src/server.rs:44"}
{"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020679041Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020684682Z","hostname":"sock","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020695982Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020702762Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020717802Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020727553Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"}
{"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.020731763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"}
{"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.023933049Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3849"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.030991116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.041167368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.051059694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.060401878Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.069421216Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.07781638Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.086192735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.094484897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.102680527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.102724208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"}
{"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.102732249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.102737369Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.102742749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.102748359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.114623026Z","hostname":"sock","pid":1555,"sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:355"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.114678057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.114686647Z","hostname":"sock","pid":1555,"sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:367"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.114767459Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 79.423477ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.120539059Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"701c516a-e11d-4925-a578-f7ae5782a9d5\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"701c516a-e11d-4925-a578-f7ae5782a9d5\" }"}
{"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.124953491Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"attempting to create encrypted zvol: name=\"oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.127652167Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:153"}
{"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.142813762Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"successfully created encrypted zvol: name=\"oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.150491102Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:249"}
{"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.150543153Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:96"}
{"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.155535337Z","hostname":"sock","pid":1555,"sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:382"}
{"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.155544227Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"}
{"msg":"Ensuring dataset oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.155549898Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.200940512Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 121.646521ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.201537384Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"85a9edee-feb2-4397-89eb-8b3d4180f52a\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"85a9edee-feb2-4397-89eb-8b3d4180f52a\" }"}
{"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.229565927Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.324696327Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 171.894585ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.325220008Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36ca5c40-66b3-458a-b380-61b68a92cb9d\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"36ca5c40-66b3-458a-b380-61b68a92cb9d\" }"}
{"msg":"Mounting oxi_c3ed08c3-e0e1-43f2-b040-3c4755a2167c/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.448450531Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"}
{"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.461435091Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.461514773Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.461542944Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"}
{"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.461555974Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.497985532Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 95.619005ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.498524813Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a004534c-1236-4bc3-bd6a-633835eb5c1f\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a004534c-1236-4bc3-bd6a-633835eb5c1f\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.595592593Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 194.830069ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.595984441Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd9f34c8-02be-4b68-884d-0c320f377d64\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bd9f34c8-02be-4b68-884d-0c320f377d64\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.7925537Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"Failed enable ddm stats (will retry after 333.055205ms","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:07.793312816Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e67acb4d-63d5-4730-9a2a-fcffedf5e699\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e67acb4d-63d5-4730-9a2a-fcffedf5e699\" }"}
{"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.127782585Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"}
{"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.159402763Z","hostname":"sock","pid":1555,"sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.160202699Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.16023263Z","hostname":"sock","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.160310701Z","hostname":"sock","pid":1555,"file":"oximeter/producer/src/lib.rs:284","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:284","interval":"30s","address":"[fd00:1122:3344:101::1]:55688","producer_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","route":"//acb2a796-8e9f-4811-b601-726a12d79b0b"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:08.167239786Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"210.883973ms"}
{"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.208877202Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"}
{"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.208907573Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"}
{"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.208978584Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:993","underlay_address":"fd00:1122:3344:101::1"}
{"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.208989514Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"}
{"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.20972573Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:196"}
{"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.20975856Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.209790031Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.209803501Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.210020536Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"}
{"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.210200609Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"}
{"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.21022725Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV1 { rack_subnet: Ipv6Network { addr: fd00:1122:3344:100::, prefix: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV1 { routes: [RouteConfig { destination: V4(Ipv4Network { addr: 0.0.0.0, prefix: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [V4(Ipv4Network { addr: 10.151.2.174, prefix: 24 })], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"}
{"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.21024269Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3617"}
{"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.210256541Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:53471","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"}
{"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.388709763Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.388775605Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.388984879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.389069721Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.389109502Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.389149972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.389166783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:08.390421129Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:08.390660594Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"464.359786ms"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.421492145Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42554"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.42169224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.421770371Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"54c00772-1052-4c33-8ab3-066a02009ede","remote_addr":"[fd00:1122:3344:101::1]:42554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":164,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.453248626Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37703"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.453346178Z","hostname":"sock","pid":1555,"uri":"/sled-role","method":"GET","req_id":"56c373a4-1220-4b81-a0a0-9544056e8f67","remote_addr":"[fd00:1122:3344:101::1]:37703","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":33,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.454832779Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.45486751Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.455205497Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.455431991Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:815"}
{"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: bf2a0dce-f659-4a90-aa0b-dd0a03b87954, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, pool_id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool) }, OmicronPhysicalDiskConfig { id: 3da5cdb7-031b-441d-bcba-8297bc53c14f, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, pool_id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool) }, OmicronPhysicalDiskConfig { id: 2da42829-b77d-42b5-8212-48a387442b7c, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, pool_id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool) }, OmicronPhysicalDiskConfig { id: d27566ee-ab55-4b03-8f38-07e4c4675371, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, pool_id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool) }, OmicronPhysicalDiskConfig { id: 6c962dd2-23d0-4373-b1a1-9279cf19ae6d, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, pool_id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool) }, OmicronPhysicalDiskConfig { id: a3e4dcc9-7f51-462f-af2f-8c74c5b2c173, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, pool_id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool) }, OmicronPhysicalDiskConfig { id: 4c36d026-3af1-492b-8432-acefaab135a6, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, pool_id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool) }, OmicronPhysicalDiskConfig { id: 2150da11-52cd-42fe-9d41-aae1a3441958, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, pool_id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool) }, OmicronPhysicalDiskConfig { id: 575ec239-a350-4c7d-8ea3-b2a988f60446, identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, pool_id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486424926Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:347"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486520638Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52314"}
{"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, id: bf2a0dce-f659-4a90-aa0b-dd0a03b87954, pool_id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, id: 3da5cdb7-031b-441d-bcba-8297bc53c14f, pool_id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, id: 2da42829-b77d-42b5-8212-48a387442b7c, pool_id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, id: d27566ee-ab55-4b03-8f38-07e4c4675371, pool_id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, id: 6c962dd2-23d0-4373-b1a1-9279cf19ae6d, pool_id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, id: a3e4dcc9-7f51-462f-af2f-8c74c5b2c173, pool_id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, id: 4c36d026-3af1-492b-8432-acefaab135a6, pool_id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, id: 2150da11-52cd-42fe-9d41-aae1a3441958, pool_id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, id: 575ec239-a350-4c7d-8ea3-b2a988f60446, pool_id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486748503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486755453Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486768013Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"}
{"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486772523Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"}
{"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486803274Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486808674Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486813644Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.486817774Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.58080294Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:08.58082563Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool)"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:08.947605361Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"877.607412ms"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:09.212580993Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.590459595Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.765434375Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.765547207Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.765555378Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.765574678Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.765585328Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.765745091Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:09.765752242Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:09.95072108Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.903407045s"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:10.325715591Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:10.519229967Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:10.736367294Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:10.736463526Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Automatically destroying dataset: oxp_4e6ef30f-a7d9-4acc-99aa-59ae1084fa14/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:10.948318674Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 61.742857ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:11.135082558Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 112.265741ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:11.669713901Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:12.281613911Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:12.281652702Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 114.940381ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:12.281678343Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 482.068348ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:12.785598927Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:12.964897317Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.84884602s"}
{"msg":"Looking for unencrypted datasets in oxp_4e6ef30f-a7d9-4acc-99aa-59ae1084fa14","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:13.61444375Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:13.847785954Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.166310241s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:13.847975478Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:13.853960123Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:13.853973053Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:13.854021904Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:13.854030214Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:13.854036724Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:14.091594826Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:14.091639817Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"12d428b9-eedd-40c0-8928-1d92a73f181d (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.280293976Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.510024645Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.510057626Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.510064336Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.510101057Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.510111457Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.510286561Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:15.510305211Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:16.281465955Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:16.764242569Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:16.997363858Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.765630932s"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:17.238775881Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:17.49588904Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Automatically destroying dataset: oxp_12d428b9-eedd-40c0-8928-1d92a73f181d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:17.754616393Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:17.986544127Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:17.986639789Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2303"}
{"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.445873794Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:18.498204822Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_12d428b9-eedd-40c0-8928-1d92a73f181d","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.563628323Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.571962347Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.572013038Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.572055569Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.572063119Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.572070239Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.582258981Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.582292842Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.678149916Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.686396768Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.686433649Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.686440039Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.6865208Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.68653198Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.686667593Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.686674303Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.721931717Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.730313262Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.855545477Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.863463152Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.863490002Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.863510423Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.863516323Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.863524983Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.873637163Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.873662874Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.96625763Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.974664814Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.974692515Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.974699195Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.974718595Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.974732936Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.974870709Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:18.974878469Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.011208585Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_7ba4af02-bf8f-47c6-af79-d69fdad5d4bf/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.019890355Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_7ba4af02-bf8f-47c6-af79-d69fdad5d4bf","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.145273454Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.153448374Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.153477454Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.153503265Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.153510045Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.153515095Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.163379391Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.163404541Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"62a187a2-0706-420e-a881-83ff781ad6c1 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.260143944Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.26812804Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.268185631Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.268196351Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.268222062Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.268232452Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.268405685Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.268415886Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.304025726Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_62a187a2-0706-420e-a881-83ff781ad6c1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.311764198Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_62a187a2-0706-420e-a881-83ff781ad6c1","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.436979203Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.44503327Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.445073301Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.445080621Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.445086901Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.445093752Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.455339515Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.455368255Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"5da68ecb-08f6-4895-925f-903c81000441 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.555492888Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.562622557Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.562662898Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.562687968Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.562699278Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.562710819Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.562877542Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.562890062Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:19.56710712Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.598942582Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_5da68ecb-08f6-4895-925f-903c81000441/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.606807006Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_5da68ecb-08f6-4895-925f-903c81000441","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.730726894Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.738754871Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.738794232Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.738817482Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.738836493Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.738844273Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.748425082Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.748450203Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.843610543Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.851056887Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.851105228Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.851113649Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.851125129Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.851143699Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.851281632Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.851294103Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:19.854887237Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.888758082Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_3b36f3f1-29d0-4d11-8b6b-ae2a89620d43/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:19.897036864Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_3b36f3f1-29d0-4d11-8b6b-ae2a89620d43","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.029959599Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.038793483Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.038844433Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.038853554Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.038862164Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.038869774Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.048525005Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.048563216Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"2780ea55-1799-4c3e-916a-9024b9586eaf (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.14105529Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.147971264Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.148008585Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.148028575Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.148036525Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.148098326Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.14825627Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.14826623Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.185684939Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_2780ea55-1799-4c3e-916a-9024b9586eaf/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.193375839Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"Looking for unencrypted datasets in oxp_2780ea55-1799-4c3e-916a-9024b9586eaf","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.314379026Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.322047045Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.322081236Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.322091497Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.322098997Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"}
{"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.322105587Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:112","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.331572024Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"}
{"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.331604394Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool)"}
{"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.426487428Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","zpool":"ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External }"}
{"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.433699248Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.433735279Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.433744019Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.43375824Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }","epoch":"0"}
{"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.43377255Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.433909333Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"}
{"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.433919813Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"}
{"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.470167977Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"}
{"msg":"Automatically destroying dataset: oxp_bae019e4-4e95-466e-aa8a-f660660e26c1/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.478105692Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:20.569263429Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Looking for unencrypted datasets in oxp_bae019e4-4e95-466e-aa8a-f660660e26c1","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.602443719Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"}
{"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.61019285Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.610251381Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.610304703Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.610487126Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: c3ed08c3-e0e1-43f2-b040-3c4755a2167c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 3907cd6d-3c6d-4543-92de-9ec01f9bc32c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"}
{"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.610551758Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.610564428Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.610691501Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.611033688Z","hostname":"sock","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"a8bd8eae-8392-415b-b726-51516711c801","remote_addr":"[fd00:1122:3344:101::1]:52314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":12124443,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.641659865Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.641809388Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50895"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.642133095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.642141945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.693389841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.693429312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.745485215Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.745514126Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.797077428Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"No ledger in /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.797113649Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"No ledger in /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.79713355Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.797185161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.842154156Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.842201447Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.842213678Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.896778863Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.900840387Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.904790729Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.917135206Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.921316383Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:20.925176413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:08:21.571063831Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:21.764885834Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:393","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:393","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"15.454121794s"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:21.853488007Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Profile for oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:2::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:2::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:2::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:2::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:22.442561572Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:22.442667914Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:22.442807777Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:3::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:3::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:3::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:3::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:22.929325009Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:23.416952784Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"}
{"msg":"Profile for oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:1::2\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:1::1\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/internal_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:1::1]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"[fd00:1122:3344:1::1]:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:23.417052386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:23.417076316Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 69.341009ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.425567163Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:23.425597734Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:23.429884423Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 138.27505ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.504772551Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 66.1532ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.54942736Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 34.756274ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.555956936Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 134.171434ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.59799071Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 120.441799ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.702076016Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 273.14767ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.759072102Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 208.055766ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.877806752Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 109.485985ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:23.898163135Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 571.647691ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:24.027296491Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 300.448797ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:24.042019477Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 480.014683ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:24.096804547Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 506.309752ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:24.350833972Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 952.571801ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:24.58624834Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 1.160549834s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:24.608599805Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 1.485934287s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:24.869274028Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 886.927132ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:25.550272906Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 1.17524673s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:25.778418892Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:26.021653012Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2320"}
{"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:26.021663302Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 754.976855ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:26.364160648Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 580.641515ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:26.445097982Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fedf:4abf","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:26.56804456Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2327"}
{"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:26.573679797Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2388"}
{"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:26.691055759Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2573"}
{"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:26.93193071Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2737"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 1.058980197s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:26.964896456Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 1.194010768s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:27.034211468Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:27.124647709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2452"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 696.935222ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:27.128652963Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:27.427063761Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2487"}
{"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:27.647800344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2787"}
{"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:27.83172013Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2814"}
{"msg":"wait for service Some(\"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982\") failed: Property not found. retry in 1.129063318s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:27.835310225Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c\") failed: Property not found. retry in 841.067879ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:28.034020168Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.048188983Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2681"}
{"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.23549107Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2481"}
{"msg":"wait for service Some(\"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62\") failed: Property not found. retry in 1.113305531s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:28.23842184Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.245975488Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3873","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.245992668Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3889"}
{"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.246019549Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"}
{"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.8017419Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3897"}
{"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.808511471Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3953"}
{"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:28.967038099Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4015"}
{"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.067623111Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3996","rack_subnet":"fd00:1122:3344::"}
{"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.088310091Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4043"}
{"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.184463672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4070"}
{"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.308468452Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:365"}
{"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.340470598Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:413"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.370685826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_adb60b2d-67ac-445a-817b-3e6611ca7982"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.370760068Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_08569023-bf34-46c3-a27b-d9b75bdbd67c"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.370769048Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_internal_dns_18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.370776888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.371051084Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.371416911Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8908c98c-00ab-42cf-93e4-be0163903f21","remote_addr":"[fd00:1122:3344:101::1]:50895","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8729531,"response_code":"204"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.373342471Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"}
{"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.379369597Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"}
{"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.379449489Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"}
{"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.379891918Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"3"}
{"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.379903218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:435","config":"PortSettings {\n    links: {\n        \"0\": LinkSettings {\n            addrs: [\n                10.151.2.174,\n            ],\n            params: LinkCreate {\n                autoneg: false,\n                fec: None,\n                kr: false,\n                lane: Some(\n                    LinkId(\n                        0,\n                    ),\n                ),\n                speed: Speed40G,\n            },\n        },\n    },\n}"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.42665102Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"}
{"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462807373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462883144Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462893545Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462902935Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462910095Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-0\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462917115Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462925485Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-M2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462954046Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462961206Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-1\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462969866Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462976656Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-2\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462984916Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462992217Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-3\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.462998717Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463004027Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-4\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463012957Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463018007Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-5\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463022767Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463027657Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-6\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463036037Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463041118Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-7\", model: \"synthetic-model-U2\" }"}
{"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463045948Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.463050728Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", serial: \"synthetic-serial-8\", model: \"synthetic-model-U2\" }"}
{"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.490691893Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:548"}
{"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.537916215Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:584"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.537927996Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.569088224Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.569714257Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.569725977Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.598142068Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.59823414Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49467"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.598380053Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.598387424Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.619440911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.619490772Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.636847064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.636888165Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.655496632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.655552483Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.655685336Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.655801028Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.665889088Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.697322192Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.697995806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.698001996Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.742300798Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 138, 18]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.742334168Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"ntp"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.742809038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.742816448Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.169,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.743066714Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.744477733Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.766498191Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:29.786388355Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/chrony-setup\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"allow\" value=\"fd00:1122:3344:100::/56\"/>\n        <propval type=\"boolean\" name=\"boundary\" value=\"true\"/>\n        <propval type=\"astring\" name=\"server\" value=\"0.pool.ntp.org\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"1.1.1.1\" />\n            <value_node value=\"9.9.9.9\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/ntp\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.3.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.3.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:30.416526574Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:30.416604046Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 30.606696ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:30.557828324Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 87.265972ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:30.595684441Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 126.830481ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:30.695881656Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 332.146718ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:30.833866177Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 1.031646728s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:31.175670647Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 894.92526ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:32.218436671Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:32.731899274Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61118"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:32.732021196Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b6a26b1f-9489-4234-9b3f-ab52232fde61","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61118","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 1.341077672s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:33.12395966Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 770.329619ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:34.474980267Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687\") failed: Property not found. retry in 1.440427843s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:35.254862723Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.715209324Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.715292126Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.715548431Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.715788016Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"afebd678-1fa6-4c83-8ea9-e243045880c4","remote_addr":"[fd00:1122:3344:101::1]:49467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7117508,"response_code":"204"}
{"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.717621284Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:625"}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.74673593Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.746906204Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59323"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.755459842Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.755647255Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"7f790f91-d603-4eab-99d6-12b3b699c435","remote_addr":"[fd00:1122:3344:101::1]:59323","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8582,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:36.758037555Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.851387807Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:36.85152518Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35565"}
{"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_952adea2-114c-4099-9bd9-86f261fe9687': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1  stdout: 506 Cannot talk to daemon\n  stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:40.859489273Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3603"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:40.859603185Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"cbc2d379-ccba-419d-bd67-7fe835760aca","remote_addr":"[fd00:1122:3344:101::1]:35565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007967,"response_code":"500"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:40.86414585Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.012341002Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.012439274Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50742"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.021973252Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"77075011-956a-497b-b697-aedd8324854b","remote_addr":"[fd00:1122:3344:101::1]:50742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9433,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.024457394Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:41.024465674Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.257593534Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.257696797Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48949"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.266649453Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"2e190019-5674-4a9a-97dd-315d68a90b50","remote_addr":"[fd00:1122:3344:101::1]:48949","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8873,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.269579704Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:41.269587784Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.425781974Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.425881536Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51419"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.433493444Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"08566412-3cb2-4b07-a06f-12c1b915bfda","remote_addr":"[fd00:1122:3344:101::1]:51419","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7534,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.436458036Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:41.436465696Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.595073776Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.595169678Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41619"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.602934319Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"a3d10a62-b7c8-4bdc-beeb-8ebc98dc75dc","remote_addr":"[fd00:1122:3344:101::1]:41619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7686,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.6058651Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:41.60587272Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.956889013Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.956984675Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61670"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.965503422Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"02b96f3f-879f-41d4-a54f-364aa12a70dc","remote_addr":"[fd00:1122:3344:101::1]:61670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8441,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:41.966992933Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:41.967008194Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.332184911Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.332289993Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53088"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.340871652Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"99df8206-c000-43e6-8edc-1e2e34ef9f1f","remote_addr":"[fd00:1122:3344:101::1]:53088","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8429,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.342444865Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:42.342450725Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.582934168Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.583067041Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:46420"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.590836523Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c0986356-7ae9-4ac3-984d-67b2857842ee","remote_addr":"[fd00:1122:3344:101::1]:46420","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7651,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:42.59262669Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:42.59263321Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.150566037Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.150667459Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45241"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.158849669Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"6b5aa85e-4891-4447-87bc-c9eab6ffb868","remote_addr":"[fd00:1122:3344:101::1]:45241","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":8094,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.160492793Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:43.160499054Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.889680024Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.889850058Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56708"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.898943457Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"19348d94-f53f-4b74-af5e-beeb5c819957","remote_addr":"[fd00:1122:3344:101::1]:56708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":9021,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:43.900717334Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:43.900723444Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:44.767895385Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:44.768029438Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64487"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:44.776121476Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"9b09a1cf-5ccc-4013-8c98-bf16360a01b0","remote_addr":"[fd00:1122:3344:101::1]:64487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":7976,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:44.777675059Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:44.777688399Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:655","error":"\"Time is synchronized on 0/1 sleds\""}
{"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.262376809Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:605"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.262479121Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35742"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.283010739Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"c5e3b249-bd4f-41c0-ada2-6762ca43e27e","remote_addr":"[fd00:1122:3344:101::1]:35742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":20393,"response_code":"200"}
{"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1163513699, ip_addr: 69.89.207.99, stratum: 2, ref_time: 1715756925.238036, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.286059442Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:633"}
{"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.286077762Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1216"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.316557606Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.31670898Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:46219"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.316908004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.316918024Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.337250397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.337314518Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.356354705Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.356405995Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.376025034Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.376073325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.436840249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.43688213Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.488666707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.488708188Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.540001465Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.540046866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.591136589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.59118112Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.640752071Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.640807752Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.640990896Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.641141319Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.673540753Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.673587754Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.673598055Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.673607775Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.673617835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.673747298Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"cb805569-83a4-4e8b-800a-1a73ad54a20b","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":356992,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.673967902Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"65.832012ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb805569-83a4-4e8b-800a-1a73ad54a20b\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb805569-83a4-4e8b-800a-1a73ad54a20b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.741833084Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.742179072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.742195692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.758809167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.758848088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.777144969Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.77717947Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.79546136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.795499391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.814484965Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.814521556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.833147753Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.833181404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.851864823Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.851909944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.870534341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.870571002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.889188769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.88923036Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.889461695Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.889616598Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.922511772Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.922553613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.922563994Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.922574634Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.922585444Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:45.922699967Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"960bdd3e-303e-46cc-8ea6-7fb7fb14fa4f","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":180698,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:45.92287593Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"149.706127ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"960bdd3e-303e-46cc-8ea6-7fb7fb14fa4f\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"960bdd3e-303e-46cc-8ea6-7fb7fb14fa4f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.073251399Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.073723558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.073742759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.094594053Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.094634164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.114221611Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.114259402Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.134198237Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.134234937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.15262301Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.152662431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.170929601Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.170963172Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.189972707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.190029258Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.208318699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.20835457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.22665868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.226698991Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.226871125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.226989247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.260065615Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.260118447Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.260131207Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.260141587Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.260152117Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.26031038Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"35d250df-3018-433a-b9b2-f1a02f4ed89f","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized)]","latency_us":186852,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.260520065Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"203.372159ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"35d250df-3018-433a-b9b2-f1a02f4ed89f\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"35d250df-3018-433a-b9b2-f1a02f4ed89f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.464897147Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.465306845Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.465322786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.484413573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.484465134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.503176323Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.503218734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.521807791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.521864952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.540180413Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.540225574Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.558828181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.558868932Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.57753761Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.577574151Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.595108246Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.595153627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.613908627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.613965078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.614151142Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.614288285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.648000056Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.648051567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.648060038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.648070378Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.648080088Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.648201581Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"cdd554ae-a04d-4415-980e-8967950ddb2b","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized)]","latency_us":183095,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.648430245Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"115.118132ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cdd554ae-a04d-4415-980e-8967950ddb2b\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cdd554ae-a04d-4415-980e-8967950ddb2b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.764804956Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.76499397Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.765009481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.783620618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.783654809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.802799727Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.802828847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.821779901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.821813581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.840238525Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.840274046Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.859414614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.859464875Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.876501729Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.87653595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.893515873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.893551304Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.91063166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.91067202Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.910857434Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.910978547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.943768549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.94382123Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.94383339Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.94385214Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.943861571Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:46.943951323Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"09293551-9be4-411a-9b90-c6e229a601c3","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized)]","latency_us":179057,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:46.944150137Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"303.627773ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"09293551-9be4-411a-9b90-c6e229a601c3\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:46 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"09293551-9be4-411a-9b90-c6e229a601c3\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.249562331Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.249947059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.249956669Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.268959064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.268999825Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.288122063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.288154504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.306550366Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.306588217Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.325354658Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.325387418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.344409254Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.344445005Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.363382979Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.36341859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.382064197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:47.382108189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.002580574Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.002615914Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.002770638Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.00289462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.036003119Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.03604973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.03606043Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.036069201Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.036079121Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.036234304Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"309cae2b-f25a-4047-bf88-9cd130250999","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized)]","latency_us":184478,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.036491969Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"317.193025ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"309cae2b-f25a-4047-bf88-9cd130250999\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"309cae2b-f25a-4047-bf88-9cd130250999\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.353870062Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.354133608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.354143558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.372546731Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.372576711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.391787351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.391826512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.409296055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.409335106Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.428927723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.428961524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.447375377Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.447414698Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.465211148Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.465244569Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.483805565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.483844885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.501704547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.501755328Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.501933472Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.502084935Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.533908917Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.533981989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.533993059Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.534002839Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.534013369Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.534119882Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e765f94e-f27f-4e1f-9865-e5413cef99c2","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":180104,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.534350716Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"162.071826ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e765f94e-f27f-4e1f-9865-e5413cef99c2\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e765f94e-f27f-4e1f-9865-e5413cef99c2\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.697727645Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.698074573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.698089853Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.716627159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.716660749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.73496223Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.735001611Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.753827573Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.753871583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.770686763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.770724404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.788139856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.788173577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.805948227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.805989608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.823621665Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.823658515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.840766541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.840815282Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.841076958Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.841249261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.873501912Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.873554333Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.873566134Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.873578424Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.873588204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:48.873666806Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2e2c06bc-a7c9-4461-b5d9-5eed21da466e","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":175780,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:48.87385499Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"496.32777ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2e2c06bc-a7c9-4461-b5d9-5eed21da466e\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2e2c06bc-a7c9-4461-b5d9-5eed21da466e\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.371283479Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.371660336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.371678257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.390220493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.390266834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.409353441Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.409394192Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.42907046Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.429116861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.448184998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.448233359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.467788895Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.467821806Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.486223399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.48626345Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.503544769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.5035875Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.520620495Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.520671266Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.520833319Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.520962382Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:49.552014108Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:49.552054529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:49.552063709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:49.552072349Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:49.552079679Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:49.552212782Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c4fe0d09-b3cf-4eb7-a452-510c355c4bdb","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":180737,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:49.552427456Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"583.687742ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c4fe0d09-b3cf-4eb7-a452-510c355c4bdb\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c4fe0d09-b3cf-4eb7-a452-510c355c4bdb\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.136832205Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.137042879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.137051599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.155455302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.155509643Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.172301013Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.172339264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.1913868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.191429671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.209451976Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.209488306Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.228118954Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.228158475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.245310742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.245351913Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.262825186Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.262863497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.280382672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.280427252Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.280626357Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.280754699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.313297306Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.313326517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.313335767Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.313343347Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.313359267Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.31349712Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a0ef122c-7a62-43ed-88c5-a4c4a78a5811","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized)]","latency_us":176555,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.313733515Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"494.119885ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0ef122c-7a62-43ed-88c5-a4c4a78a5811\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a0ef122c-7a62-43ed-88c5-a4c4a78a5811\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.808942107Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.809222543Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.809235073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.828569865Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.828611026Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.846683582Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.846721733Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.866043555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.866084266Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.883355495Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.883394406Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.902213868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.902254349Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.919502847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.919542188Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.937157385Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.937201376Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.955927665Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.955985026Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.95618238Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.956332403Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.988612255Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.988668856Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.988681207Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.988693117Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.988703797Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:50.988796789Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bee4f337-00de-4ea4-bfa1-e2ba2a9dc547","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":179716,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:50.988995683Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"622.18083ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bee4f337-00de-4ea4-bfa1-e2ba2a9dc547\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bee4f337-00de-4ea4-bfa1-e2ba2a9dc547\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.612105816Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.612494124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.612519694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.633808837Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.633843118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.653257092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.653297603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.672666826Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.672699197Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.694245595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.694279995Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.714009636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.714062127Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.732669344Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.732705485Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.751942235Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.751977996Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.771278457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.771325569Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.771545393Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.771697666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:51.80551535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:51.805565181Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:51.805591191Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:51.805601842Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:51.805611722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:51.805698524Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f9479879-cc41-4b37-8658-dfcce586acd4","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized)]","latency_us":193369,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:51.805873087Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"802.125606ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9479879-cc41-4b37-8658-dfcce586acd4\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:50 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9479879-cc41-4b37-8658-dfcce586acd4\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.609327372Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.609770971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.609782592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.629779838Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.629832869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.648341214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.648392515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.667556794Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.667602555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.686742973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.686780154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.705142436Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.705181136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.724484778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.724529909Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.743269429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.74331383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.76207009Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.762118851Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.762346096Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.762525939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:52.796292762Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:52.796342033Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:52.796351953Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:52.796362534Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:52.796372374Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:52.796489836Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"30c7abaf-6fbc-49a3-9280-e2d1ac8e5cf8","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":186985,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:52.796706361Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"720.347015ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30c7abaf-6fbc-49a3-9280-e2d1ac8e5cf8\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"30c7abaf-6fbc-49a3-9280-e2d1ac8e5cf8\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.518242182Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.519287144Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.519292364Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.538916472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.538965373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.55850852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.55854302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.57678967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.576828721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.595618672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.595679353Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.614122817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.614163898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.632092931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.632136371Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.649416551Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.649456852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.668448817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.668517908Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.668732433Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.668878136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:53.70077826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:53.700828491Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:53.700839251Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:53.700850681Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:53.700860251Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:53.700968203Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f810d3be-09e4-4edd-8da5-f9f4f8ad5a2d","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":182559,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:53.701163948Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"981.080084ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f810d3be-09e4-4edd-8da5-f9f4f8ad5a2d\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f810d3be-09e4-4edd-8da5-f9f4f8ad5a2d\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.683706139Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.684138717Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.684165618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.704744936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.704792147Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.724004927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.724061588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.743592184Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.743644835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.763367416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.763409917Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.782622226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.782669867Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.801732274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.801788085Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.820221659Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.82026362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.839334646Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.839395947Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.839631663Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.839764475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:54.873252782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:54.873291823Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:54.873300583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:54.873309163Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:54.873316033Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:54.873457306Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"be5c5b64-aaf1-4b3a-a770-0cd86853563f","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized)]","latency_us":189556,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:54.873696421Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.679257772s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"be5c5b64-aaf1-4b3a-a770-0cd86853563f\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"be5c5b64-aaf1-4b3a-a770-0cd86853563f\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:56.341474457Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"166.451793ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.554219743Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.554563481Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.554577111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.576203671Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.576255732Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.595223007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.595267977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.614287813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.614334804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.633231987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.633283228Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.652692172Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.652748733Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.671278029Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.671325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.689655431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.689711872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.708897121Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.708956533Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.709153917Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.70930301Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:56.743078463Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:56.743122444Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:56.743130484Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:56.743141004Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:56.743151554Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:56.743321918Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6d962ca5-9034-422f-a2b2-e036abeb0d71","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized)]","latency_us":188960,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:56.743565853Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"1.228987941s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6d962ca5-9034-422f-a2b2-e036abeb0d71\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6d962ca5-9034-422f-a2b2-e036abeb0d71\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:57.973147853Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:57.974391289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:57.974401829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:57.995847566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:57.995906097Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.016410653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.016457264Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.035802087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.035841618Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.055398174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.055449915Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.074751737Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.074800878Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.093466817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.093512947Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.113434552Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.113475953Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.132190402Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.132245813Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.132430467Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.13256069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:58.166506236Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:58.166579758Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:58.166591578Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:58.166601588Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:58.166612018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:08:58.166736631Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c19a2e72-69e2-4673-82f6-299687639c71","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":193379,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:08:58.166939445Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.594466711s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c19a2e72-69e2-4673-82f6-299687639c71\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:08:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c19a2e72-69e2-4673-82f6-299687639c71\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.762766149Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.763127466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.763147026Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.783477349Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.78352415Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.80321553Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.803255821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.821730015Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.821776566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.842503057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.842541108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.86091804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.860950591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.87961434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.879664901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.898834289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.89887176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.917436596Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.917491437Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.917732813Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.917863945Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:00.950029305Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:00.950088546Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:00.950100806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:00.950110926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:00.950117006Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:00.950195698Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"11266bbc-2630-4db7-adf9-bced0d5687ee","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized)]","latency_us":187276,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:00.950398152Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"2.481282364s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"11266bbc-2630-4db7-adf9-bced0d5687ee\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"11266bbc-2630-4db7-adf9-bced0d5687ee\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.209545643Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47579"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.209692656Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d0393f44-c042-4cef-93a6-09985ad997e4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47579","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":72,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.432211705Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.432720886Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.432734206Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.453519118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.45357249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.473848271Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.473894672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.49350953Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.493556601Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.513623899Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.51366567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.533022172Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.533065793Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.551723921Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.551764672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.571444802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.571482943Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.590215022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.590256833Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.590452877Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.59059205Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:03.624796632Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:03.624839923Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:03.624850453Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:03.624858233Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:03.624867953Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:03.624978226Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6632bf65-1d4d-4339-a9cc-462c12182c40","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized)]","latency_us":192497,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:03.62517006Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"3.096175391s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6632bf65-1d4d-4339-a9cc-462c12182c40\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6632bf65-1d4d-4339-a9cc-462c12182c40\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.722372045Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.722776093Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.722808664Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.745359843Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.745413944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.765637325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.765678796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.784404816Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.784449447Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.804500724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.804534925Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.823823146Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.823857056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.841843871Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.841883371Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.861243844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.861281355Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.880690919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.8807329Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.880922414Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.881059166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:06.916596046Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:06.916651517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:06.916659037Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:06.916667757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:06.916674427Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:06.91677232Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"50753cb7-ee78-4b91-8bce-132c0612f5a8","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":194219,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:06.917000574Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"3.294517183s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"50753cb7-ee78-4b91-8bce-132c0612f5a8\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"50753cb7-ee78-4b91-8bce-132c0612f5a8\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.212296831Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.212673498Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.212687759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.233862999Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.23389885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.254259444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.254303365Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.272784159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.27282802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.290649861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.290693462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.310257429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.31029442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.328164881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.328203162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.347019604Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.347054634Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.364687531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.364739472Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.365012768Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.365174091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:10.397748469Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:10.39779089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:10.39780107Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:10.39781114Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:10.39782249Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:10.397922603Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"badd8074-ffbb-4e77-9958-593538c0a44f","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":185466,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:10.398132027Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"3.974484717s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"badd8074-ffbb-4e77-9958-593538c0a44f\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"badd8074-ffbb-4e77-9958-593538c0a44f\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:11.542930583Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"654.623942ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.373148664Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.374653916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.374661176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.396311716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.396352697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.416069766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.416103357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.435930159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.43597217Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.454995106Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.455038847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.474461601Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.474499742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.493079059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.49312726Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.512192546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.512237967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.531442786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.531486318Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.531678391Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.531818545Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:14.564918483Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:14.564973284Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:14.564983344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:14.564993325Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:14.565003475Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:14.565102717Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"590619cc-fa01-4d79-91f3-43e20261f66b","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized)]","latency_us":191798,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:14.565307061Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"5.888114464s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"590619cc-fa01-4d79-91f3-43e20261f66b\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"590619cc-fa01-4d79-91f3-43e20261f66b\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.454246696Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.455236836Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.455254417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.477437298Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.477486639Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.497559997Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.497598698Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.516097543Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.516142064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.535115228Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.535157849Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.55346971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.553511351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.572466775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.572506396Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.590777986Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.590819367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.608700159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.60874051Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.608948054Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.609086487Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:20.641060112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:20.641104813Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:20.641115493Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:20.641146324Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:20.641157514Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:20.641303408Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dc21f78c-1282-451b-89d5-5c288e9ab8e1","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized)]","latency_us":186881,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:20.641532882Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"4.32672259s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dc21f78c-1282-451b-89d5-5c288e9ab8e1\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dc21f78c-1282-451b-89d5-5c288e9ab8e1\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:24.969297949Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:24.969777999Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:24.96980072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:24.991331257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:24.991380749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.011434456Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.011492717Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.03039179Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.030433491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.048539198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.048581559Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.068690857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.068725318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.086752663Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.086787794Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.106442632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.106484693Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.124828065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.124868826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.125110451Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.125272724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:25.159465435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:25.159509876Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:25.159520537Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:25.159532567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:25.159542737Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:25.159654839Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a7738156-be03-48c6-a58c-b1cf055c4c03","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized)]","latency_us":190140,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:25.159845114Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"7.9695987s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a7738156-be03-48c6-a58c-b1cf055c4c03\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a7738156-be03-48c6-a58c-b1cf055c4c03\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:27.230574793Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"648.441642ms"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.129951346Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.130250832Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.130258562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.153453785Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.153499486Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.174013312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.174058283Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.193735863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.193778654Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.213598736Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.213638937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.233634203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.233674734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.242456356Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35017"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.24262731Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e17b0547-f814-4e8e-8ace-a5e1294af997","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:35017","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.252955165Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.253001816Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.272067003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.272110403Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.290960405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.291003346Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.2911806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.291301723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:33.324848721Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:33.324912712Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:33.324923242Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:33.324932872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:33.324944323Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:33.325035405Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f3b51397-f65b-48e2-82fd-3553e1e78d2f","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized)]","latency_us":194942,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:33.325224418Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"6.279875667s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f3b51397-f65b-48e2-82fd-3553e1e78d2f\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f3b51397-f65b-48e2-82fd-3553e1e78d2f\" }"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.606858543Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.607998546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.608031827Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.63026245Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.630304851Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.650160334Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.650198315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.669607128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.669649659Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.689607294Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.689659096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.708712792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.708749583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.7273867Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.727432461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.746485248Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.746526159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.764960542Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.765010373Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.765186737Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.765307879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:39.799285256Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:39.799338557Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:39.799349447Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:39.799359558Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:39.799369518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:39.799500721Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bd23c28d-fcea-4ce3-a07e-a3d37f7bb73d","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized)]","latency_us":192427,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:39.799738406Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"12.93951738s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bd23c28d-fcea-4ce3-a07e-a3d37f7bb73d\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bd23c28d-fcea-4ce3-a07e-a3d37f7bb73d\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:42.913636868Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"1.036520077s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.803357554Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.803757372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.803772603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.826384013Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.826440284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.845558652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.845592743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.864078507Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.864124808Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.8819909Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.882028941Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.900554006Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.900587797Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.918415858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.918461359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.93631087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.936344161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.954248153Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.954281274Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.95454403Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.954654442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:52.988363713Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:52.988409964Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:52.988420414Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:52.988430795Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:52.988440785Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:09:52.988530046Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"52227679-a170-4104-9267-bb76fbea8e80","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":185022,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:52.98871555Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"8.985632361s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"52227679-a170-4104-9267-bb76fbea8e80\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:09:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"52227679-a170-4104-9267-bb76fbea8e80\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:09:59.031378774Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"5.907476941s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:01.951317726Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:01.951623022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:01.951634833Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:01.972266903Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:01.972309214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:01.992274001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:01.992306081Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.01238928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.012417121Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.030149011Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.030179321Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.050566537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.050597727Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.068795647Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.068835108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.087100329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.08715541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.105459782Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.105510193Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.105713537Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.10583677Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:02.137241565Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:02.137288096Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:02.137298916Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:02.137309926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:02.137320116Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:02.137450879Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d1a65eb4-8eab-4f08-9485-288e93121900","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized)]","latency_us":186515,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:02.137702195Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"14.726985305s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d1a65eb4-8eab-4f08-9485-288e93121900\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:10:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d1a65eb4-8eab-4f08-9485-288e93121900\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:03.311891729Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48092"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:03.312008619Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d532c044-f0b2-4dc7-ada1-548a4203a694","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48092","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":62,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.825787219Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.826710468Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.826739839Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.847060992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.847104323Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.86658802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.86661422Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.884257698Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.884288509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.903814846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.903838387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.923234972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.923265222Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.943622187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.943650057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.962027357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.962055181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.980243251Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.980301312Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.980510826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:16.980648779Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:17.012651057Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:17.012699228Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:17.012709818Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:17.012720068Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:17.012730078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:17.01282529Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"222904d9-f1cd-4938-842c-ae8b29a1b6e3","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized)]","latency_us":187402,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:17.013005674Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"8.884581932s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"222904d9-f1cd-4938-842c-ae8b29a1b6e3\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:10:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"222904d9-f1cd-4938-842c-ae8b29a1b6e3\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:19.920286451Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"6.250521478s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.874157012Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.874462468Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.874502759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.897702893Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.897744673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.918486986Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.918521227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.937948452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.937988833Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.957350777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.957392918Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.975983446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.976039327Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.994627965Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:25.994677996Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:26.012586619Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:26.012642201Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:26.031182357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:26.031241638Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:26.031477554Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:26.031625537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:26.064106844Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:26.064150915Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:26.064174125Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:26.064183606Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:26.064192316Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:26.064294178Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9e6652a9-7858-47c6-a022-ed7c9ce2ffd3","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":190529,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:26.064495212Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"11.853351532s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9e6652a9-7858-47c6-a022-ed7c9ce2ffd3\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:10:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9e6652a9-7858-47c6-a022-ed7c9ce2ffd3\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:33.262791613Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45586"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:33.262877565Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7141a425-6bde-4eff-b697-c1954bf104dc","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45586","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.886745691Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.887760142Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.887766662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.909069096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.909115567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.928827799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.928852639Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.948164602Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.948187952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.967974145Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.968014416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.987902811Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:37.987927731Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.008351198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.008381278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.027042428Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.027068038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.046195117Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.046232118Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.046550294Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.046705638Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:38.079744057Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:38.079783258Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:38.079793128Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:38.079804318Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:38.079810718Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:38.07989649Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"87e360d0-8300-42ab-9b80-00ad1d96f466","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized)]","latency_us":193536,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:38.080032913Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"13.888696225s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"87e360d0-8300-42ab-9b80-00ad1d96f466\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:10:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"87e360d0-8300-42ab-9b80-00ad1d96f466\" }"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:41.151042556Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"23.476752366s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.932837391Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.933168518Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.933180668Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.954698937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.954741478Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.972254463Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.972306295Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.990092426Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:51.990136656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.007711643Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.007747544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.02532315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.025368842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.042715793Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.042754114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.060113636Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.060149547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.07754877Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.077602741Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.077923118Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.078093631Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:52.109330143Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:52.109387484Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:52.109399864Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:52.109412005Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:52.109423905Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3030","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:10:52.109516277Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4af6b0ca-c76e-4ca5-9cda-41b92be9948c","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\", TimeNotSynchronized), (\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\", TimeNotSynchronized), (\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\", TimeNotSynchronized), (\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\", TimeNotSynchronized), (\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\", TimeNotSynchronized)]","latency_us":177019,"response_code":"500"}
{"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:10:52.109711861Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:468","retry_after":"29.427935156s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4af6b0ca-c76e-4ca5-9cda-41b92be9948c\", \"content-length\": \"124\", \"date\": \"Wed, 15 May 2024 07:10:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4af6b0ca-c76e-4ca5-9cda-41b92be9948c\" }"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:03.216043075Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46512"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:03.216145308Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a1ede15f-1eae-4837-b47b-ca4642768282","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46512","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":73,"response_code":"200"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:19.55961182Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"29.914242547s"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.459676767Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.459990404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.460001934Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.482873421Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.482923623Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.503335708Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.503377732Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.522686562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.522723603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.541824981Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.541858822Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.560061922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.560106282Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.579097559Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.579140208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.597067464Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.597116592Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.614925266Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.614979047Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.615278704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.615414876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.697951038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.69802146Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.69803079Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.69804156Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.6980491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.78864825Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.793111463Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.797563482Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.801715873Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.806198066Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.810832423Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.815304606Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.819721338Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.824227983Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:21.828994872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService9\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::7\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.586802211Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.586935554Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService6\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::4\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::4\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.61793055Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.618058713Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService7\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::5\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.670349383Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.670464806Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService8\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::6\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.700331709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.700463762Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::3\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/cockroachdb\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::3\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32221\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.707154381Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:24.707294234Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 25.957277ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:24.739837873Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 30.158492ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:24.760965734Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 98.91011ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:24.774985606Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 110.053605ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:24.79960495Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 26.200463ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:24.830397922Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 118.247142ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:24.874656176Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 208.97809ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:24.890193767Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 126.928496ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:25.002940702Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 55.062582ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:25.949397666Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 255.533843ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:26.008469778Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 62.558119ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:26.656132579Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 341.651539ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.235966035Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 140.64136ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.45050458Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 316.493892ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.501453573Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 538.016106ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.518881527Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 103.732415ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.534123295Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 658.982117ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.589428328Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 158.318981ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.602359038Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 218.258222ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.6494017Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 310.254095ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.772360714Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 563.716868ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.82823001Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 314.45902ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:27.877860985Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 749.94118ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.069047904Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 822.716309ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.093786Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 433.775582ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.202479317Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 1.259350939s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.255940382Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.472003621s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.399333033Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 672.026583ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.64415174Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 1.370054446s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.826446493Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 841.249836ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:28.923206162Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 937.460185ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:29.323556352Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 1.35119037s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:29.522050213Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 557.166031ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:29.7716144Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.337678125s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:29.875290953Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 1.368348792s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:30.203238534Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 1.042648909s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:30.267671577Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 1.205500267s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:30.335951291Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 952.148133ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:30.878927829Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.159055229s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:31.218007351Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 1.435099279s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:31.315661579Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 676.030687ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:31.54724076Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 1.215315155s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:31.576805736Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 1.265614948s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:31.837030515Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 584.997008ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:32.231591095Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.253841954s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:32.384334442Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 1.148210669s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:32.756899551Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 578.659749ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:32.798265917Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 500.754411ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:32.825062676Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 1.189335243s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:33.109274925Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:33.169477971Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55222"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:33.169599463Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"811fb532-33e0-40c4-a521-fef1ba63c7b2","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55222","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":73,"response_code":"200"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 1.417304303s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:33.335880751Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 653.437501ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:33.386361434Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 562.716695ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:33.648179136Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 919.566228ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:33.913180044Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 1.361696063s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:34.048934146Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 671.668595ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:34.218888602Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 1.03296041s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:34.302429125Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 1.325834561s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:34.75856288Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 1.399986764s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:34.838983858Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 662.064363ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:34.896201272Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 1.292759464s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:35.341641643Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 1.436275437s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:35.417192829Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.449790263s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:35.568766041Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015\") failed: Property not found. retry in 1.366299925s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:36.092127159Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3\") failed: Property not found. retry in 1.156617879s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:36.244529448Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364\") failed: Property not found. retry in 1.063676424s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:36.64057269Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14\") failed: Property not found. retry in 1.413317327s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:36.861254844Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.030118813s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:37.024672503Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 856.609269ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:38.062799939Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.079181332s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:38.928551519Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.216604685s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:40.017639441Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 894.910559ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:41.915526024Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 754.315926ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:42.820981661Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 715.746732ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:43.586700862Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 1.099954363s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:44.311707815Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 786.999032ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:45.421938452Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 591.426581ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:46.218772199Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9\") failed: Property not found. retry in 756.991228ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:11:46.819843854Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.597591565Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_2ec6ec59-0609-4ca3-a298-dfc71ccab015"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.597691387Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_9b6e3ef6-b14c-455b-8400-1ee63ad5adb9"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.597705287Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_c7d123b0-1fe2-4336-ad94-79216ae10c14"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.597720588Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_9035da45-fa42-447f-985d-89ad334749a3"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.597733528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_cockroachdb_76c39d73-e744-4aa3-9c13-dc1e7e8eb364"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.597753168Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.598361201Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.707731306Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"985021d9-8241-43da-9ea6-6213bf73e147","remote_addr":"[fd00:1122:3344:101::1]:46219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":26295351,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.739388225Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54422"}
{"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:47.767130652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3437"}
{"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:54.294676111Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3455"}
{"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.073175399Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3478"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.073294101Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"0927ca2a-e7e8-4b09-ad02-9dd0b8f66b7d","remote_addr":"[fd00:1122:3344:101::1]:54422","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":10333757,"response_code":"204"}
{"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: adb60b2d-67ac-445a-817b-3e6611ca7982, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 952adea2-114c-4099-9bd9-86f261fe9687, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: f2afc369-fa05-4e43-9c24-aaf26564f459, kind: Service { id: 952adea2-114c-4099-9bd9-86f261fe9687 }, name: Name(\"ntp-952adea2-114c-4099-9bd9-86f261fe9687\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 18])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.3.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: c7d123b0-1fe2-4336-ad94-79216ae10c14, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9035da45-fa42-447f-985d-89ad334749a3, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }, OmicronZoneConfig { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: e5538333-fcad-480c-9181-aa7e94aaf834, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 99861af6-438d-4bef-ba93-632645b46d2f, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7d8b5024-88b0-4173-8867-4e833f05550e, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 7ce930b4-1bea-41ce-8c9a-2aa16fd16d05, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 638e5dab-76c4-4b78-b15a-a7faf399d1d8, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: 42687f02-f9e0-4e2e-8b1e-8fcc2648d21d, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: ae891598-60c4-47ca-bd0b-302a9e60a0bc, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7b766ca5-5d0c-459d-bc67-f339bcac986b, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External } } } }, OmicronZoneConfig { id: 03e9085d-76ce-4798-9d35-12ceecd4c607, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0bae9f0-075a-41e1-a5e1-22133ae05550, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d62b80e0-a2d7-4b05-8ee7-7f6a9516881e, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External } } } }, OmicronZoneConfig { id: d41b4327-a68c-4bde-a48b-cfcb75b4c054, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External } } } }, OmicronZoneConfig { id: ecac0828-7786-4503-af9f-835e69870288, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External } } } }, OmicronZoneConfig { id: eece7197-e5eb-4531-85f7-6700f07faf6b, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External } } } }, OmicronZoneConfig { id: c06cd1db-bb07-41fa-af89-0fd52ff0a0a8, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.135750481Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:430"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.135905534Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60075"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.136242521Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.136249101Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 18c3fbe6-7d91-49c3-bac0-64bc0ebaaa62, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.160526896Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.160572967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: adb60b2d-67ac-445a-817b-3e6611ca7982, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.182634896Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.182680487Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 08569023-bf34-46c3-a27b-d9b75bdbd67c, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: InternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.202975408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.203011099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2ec6ec59-0609-4ca3-a298-dfc71ccab015, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.223946665Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.223989805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b6e3ef6-b14c-455b-8400-1ee63ad5adb9, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.244453181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.244498612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c7d123b0-1fe2-4336-ad94-79216ae10c14, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.264784664Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.264824435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9035da45-fa42-447f-985d-89ad334749a3, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.284589111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.284627362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 76c39d73-e744-4aa3-9c13-dc1e7e8eb364, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.304699155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.304759286Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e5538333-fcad-480c-9181-aa7e94aaf834, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.378869648Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.378923429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e5538333-fcad-480c-9181-aa7e94aaf834, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7ce930b4-1bea-41ce-8c9a-2aa16fd16d05, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.454344208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.454390069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7ce930b4-1bea-41ce-8c9a-2aa16fd16d05, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ae891598-60c4-47ca-bd0b-302a9e60a0bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.527082641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.527130852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ae891598-60c4-47ca-bd0b-302a9e60a0bc, dataset_name: DatasetName { pool_name: ZpoolName { id: 4e6ef30f-a7d9-4acc-99aa-59ae1084fa14 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7b766ca5-5d0c-459d-bc67-f339bcac986b, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.598382435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.598424875Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7b766ca5-5d0c-459d-bc67-f339bcac986b, dataset_name: DatasetName { pool_name: ZpoolName { id: 12d428b9-eedd-40c0-8928-1d92a73f181d (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 03e9085d-76ce-4798-9d35-12ceecd4c607, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.667237317Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.667292338Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 03e9085d-76ce-4798-9d35-12ceecd4c607, dataset_name: DatasetName { pool_name: ZpoolName { id: 08d4507c-e72e-4afd-9f03-5e28f4ad0909 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0bae9f0-075a-41e1-a5e1-22133ae05550, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.745955025Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.746003686Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0bae9f0-075a-41e1-a5e1-22133ae05550, dataset_name: DatasetName { pool_name: ZpoolName { id: 7ba4af02-bf8f-47c6-af79-d69fdad5d4bf (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d62b80e0-a2d7-4b05-8ee7-7f6a9516881e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.819449084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.819497095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d62b80e0-a2d7-4b05-8ee7-7f6a9516881e, dataset_name: DatasetName { pool_name: ZpoolName { id: 62a187a2-0706-420e-a881-83ff781ad6c1 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d41b4327-a68c-4bde-a48b-cfcb75b4c054, dataset_name: DatasetName { pool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.894884393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.894927904Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d41b4327-a68c-4bde-a48b-cfcb75b4c054, dataset_name: DatasetName { pool_name: ZpoolName { id: 5da68ecb-08f6-4895-925f-903c81000441 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ecac0828-7786-4503-af9f-835e69870288, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.961649942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:58.961695203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ecac0828-7786-4503-af9f-835e69870288, dataset_name: DatasetName { pool_name: ZpoolName { id: 3b36f3f1-29d0-4d11-8b6b-ae2a89620d43 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eece7197-e5eb-4531-85f7-6700f07faf6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.030314031Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.030359922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: eece7197-e5eb-4531-85f7-6700f07faf6b, dataset_name: DatasetName { pool_name: ZpoolName { id: 2780ea55-1799-4c3e-916a-9024b9586eaf (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c06cd1db-bb07-41fa-af89-0fd52ff0a0a8, dataset_name: DatasetName { pool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.097915597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.097972079Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c06cd1db-bb07-41fa-af89-0fd52ff0a0a8, dataset_name: DatasetName { pool_name: ZpoolName { id: bae019e4-4e95-466e-aa8a-f660660e26c1 (zpool), kind: External }, kind: Crucible }, responder: ... }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.166990164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.167043325Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.167344672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.167520126Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.286544031Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.299412659Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.341125257Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.341315791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.341340082Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.341371892Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.341416513Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.341483594Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.34270446Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.357447737Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.357493928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591265241Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:259"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591478856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591495186Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591504516Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591524537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591556207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591578328Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.591601518Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.592035707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.597405739Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.602496195Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.607243754Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.612002703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.618174631Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.623438901Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.774975783Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.779926166Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.784167184Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.788896533Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.79356518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.79886203Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.803245381Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.80891801Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.814517246Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.819234124Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.824572935Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.831605802Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.837998344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.844287605Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.851017255Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.856836026Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.862245949Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.868460788Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.874832721Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.880756704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.887124417Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:11:59.946921851Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.050549966Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.163853334Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.220978061Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.225341322Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.225363113Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.353517389Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 222, 14]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.353607351Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.356480461Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.356504621Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.464697022Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 234, 140]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.464752983Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.520467122Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:276","addr":"fd00:1122:3344:101::2"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.522811011Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.522828041Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.629901579Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 204, 142]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.62996373Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.634051905Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.634081176Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.76982756Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 254, 70]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.769900091Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"external_dns"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.771944184Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.771960494Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.774226232Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.774242462Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.777679793Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:301"}
{"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.777695334Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:193","addrs":"{Switch0: fd00:1122:3344:101::2}"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.865515771Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 207, 1]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"}
{"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.865576132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1290","zone_type":"nexus"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.866237946Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.866255626Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.866542872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.866558022Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.166,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.866860549Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.167,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.869413112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.872007776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.872030876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.165,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.87222192Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.164,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.874953617Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.877457369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:00.87747008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.042043953Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.051036511Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"no nat entry found for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.061287624Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"}
{"msg":"creating nat entry for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.061349195Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.061879276Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Configuring new Omicron zone: oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.071712521Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"creation of nat entry successful for: V4(\n    Ipv4Network {\n        addr: 10.151.2.168,\n        prefix: 32,\n    },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.080606116Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.083291731Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Installing Omicron zone: oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.127474111Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.136359646Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.146548188Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Installing Omicron zone: oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.155891722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Configuring new Omicron zone: oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.164688745Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:01.218188307Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:03.456608735Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42916"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:03.461342164Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ac69487d-a473-4478-b014-8079d6ea2a84","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42916","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":5974,"response_code":"200"}
{"msg":"Profile for oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService16\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::10\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::10\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.216676773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.232524063Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService13\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::14\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_08d4507c-e72e-4afd-9f03-5e28f4ad0909/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::14\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"3a0d11c9-b2a2-4a79-876b-36faa9fb4d9f\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.243621463Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.246289939Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService12\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::13\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_12d428b9-eedd-40c0-8928-1d92a73f181d/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::13\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"17ecf68f-f3fc-4c24-b9d0-9bbad0bb0f5f\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.265715073Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.266735475Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService11\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::f\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::f\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.312153689Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.313077469Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService20\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::11\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/pantry\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::11\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"17000\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.574269723Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.574941537Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService19\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::1a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_bae019e4-4e95-466e-aa8a-f660660e26c1/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1a\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"8e2c2ebf-8081-4938-81e8-8c81b0b67a2f\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.716578593Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.716802238Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService17\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::17\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_5da68ecb-08f6-4895-925f-903c81000441/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::17\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"979acc2a-4937-4b2a-9089-a4c17553a937\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.736983008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.73854611Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService22\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::15\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_7ba4af02-bf8f-47c6-af79-d69fdad5d4bf/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::15\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"089c3f0c-afa2-449f-8ea8-924037b9db1f\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.758516956Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.760036167Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 70.913775ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:05.846850404Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 58.201696ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:05.881193658Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 65.088482ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:05.909562768Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 35.321707ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:05.942149996Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService18\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::19\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_2780ea55-1799-4c3e-916a-9024b9586eaf/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::19\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"36afc74d-4c2b-4c9a-aec0-794dbba4f078\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:05.945038567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 109.563332ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:05.971465886Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 98.091036ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.093952934Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService14\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::16\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_62a187a2-0706-420e-a881-83ff781ad6c1/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::16\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"5b5fcf9b-c81b-46f9-b73e-97c320d093b8\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:06.094530257Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 57.054824ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.13460343Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:06.134779284Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_ecac0828-7786-4503-af9f-835e69870288:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService21\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::18\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_3b36f3f1-29d0-4d11-8b6b-ae2a89620d43/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::18\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"4a530305-da63-4a10-bd54-fea68adbdc01\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:06.200366108Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 286.166615ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.232605649Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 124.811779ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.26248281Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 288.701945ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.300990121Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:06.301157765Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Profile for oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService15\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::12\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/crucible/agent\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"dataset\" value=\"oxp_4e6ef30f-a7d9-4acc-99aa-59ae1084fa14/crucible\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::12\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"32345\"/>\n        <propval type=\"astring\" name=\"uuid\" value=\"0f7f620d-0605-4161-b644-f2b21242487b\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:06.327755778Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 249.772146ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.377898511Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 36.638939ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.465642227Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 67.040375ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.579193129Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 48.163472ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.610559732Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:06.610626663Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 61.501736ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.6594921Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 490.920959ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.691287651Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 285.551762ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.723235066Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 321.40866ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.752832872Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:06.753740381Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 527.986927ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.804233491Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 63.01472ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.844533429Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 31.057474ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.875192007Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 134.427809ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.924290729Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 50.379648ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:06.969898168Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 91.018381ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.021313077Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 442.077016ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.08727682Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 828.884504ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.102285532Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 40.856196ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.117240643Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 147.274322ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.134926681Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 61.250375ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.150887873Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 213.765742ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.166086429Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 199.533726ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.183543842Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 58.902693ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.197744158Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 130.59219ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.212502345Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 480.695523ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.225190929Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 79.090983ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.237163748Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 264.09353ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.276706951Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 88.294372ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.333257537Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService10\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::d\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/oximeter\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"id\" value=\"7d8b5024-88b0-4173-8867-4e833f05550e\"/>\n        <propval type=\"astring\" name=\"address\" value=\"[fd00:1122:3344:101::d]:12223\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:07.333965512Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 101.591147ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.347364641Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 29.728007ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.367931649Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 843.951732ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.391142892Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 240.012902ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.408708767Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 540.691348ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.442166593Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 378.626418ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.498571276Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 160.296603ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.529586702Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 228.176797ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.553979629Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:07.554425109Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 611.97574ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.592107893Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 550.99456ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.626989398Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 127.664502ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.648481725Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 207.599915ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.669194966Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 800.725519ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.693721287Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 208.371936ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.713381405Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 1.063199578s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.732123075Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 145.467796ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.802551221Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.050813951s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.819527334Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 565.227514ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:07.926423658Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 28.217488ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.01209784Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 444.229363ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.040693635Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 838.698203ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.072929896Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 145.201977ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.093512224Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 1.349334644s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.111979968Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 251.399211ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.127585103Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 478.35567ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.145144258Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 859.331364ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.198941686Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 1.063639161s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.224923217Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 1.039199175s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.253119554Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 160.392533ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.271935565Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 721.282628ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.400709254Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 441.426171ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.497145151Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService26\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::9\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte3\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::9]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.6:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:08.498204313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService27\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::8\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte4\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.1.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.1.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/external_dns\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"http_address\" value=\"[fd00:1122:3344:101::8]:5353\"/>\n        <propval type=\"astring\" name=\"dns_address\" value=\"172.30.1.5:53\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:08.500684514Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 1.028210724s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.521749433Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 857.969589ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.539067523Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 1.453556572s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.557069217Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:08.557317782Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:08.564497152Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 822.642841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.645279003Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 1.015358733s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.812347948Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 44.48971ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.883338715Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 68.036118ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.897292006Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 906.113497ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.912704396Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 950.267308ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.930908595Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 95.580084ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.9455884Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 749.823705ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:08.963048434Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 53.944077ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.025575875Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 240.49042ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.056748183Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 1.151980058s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.071426678Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 233.125045ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.090040456Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 1.382385141s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.137245768Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 997.517618ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.331006288Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 443.243693ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.356135711Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 475.133711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.383182594Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 684.03519ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.402499396Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 760.131343ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.418432227Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 1.176744391s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.481738784Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 925.496001ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.499400221Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 1.369378606s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.566707152Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService25\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::b\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte2\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.6\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:09.692710163Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService24\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::a\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte1\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.5\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:09.693186563Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Profile for oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService28\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::c\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/opte-interface-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"interface\" value=\"vopte5\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"172.30.2.1\"/>\n        <propval type=\"astring\" name=\"ip\" value=\"172.30.2.7\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/nexus\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:09.693228394Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:09.693564301Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:09.703164611Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:09.711987244Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 1.372029068s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.738319472Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 401.029705ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.822150056Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.172856053s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.840349155Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 933.29296ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.86174719Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 789.186134ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.891034309Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 809.355217ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:09.935147007Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 1.082991259s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.037412695Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 56.129409ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.079350717Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 1.162919272s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.107194927Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 48.491294ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.122395003Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 60.223385ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.138989968Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 54.990004ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.25446315Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 1.433055217s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.472480345Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 1.486001911s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.531243258Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 1.364949824s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.5876058Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 108.476781ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.609152319Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 52.780146ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.650237003Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 680.299636ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.667236797Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 968.209085ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.683561107Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 645.662698ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.698646761Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 59.522041ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.715877419Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 1.090361916s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.731045515Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 853.412333ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.746854954Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 286.011784ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.761311805Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 279.648672ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.775558781Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 927.985831ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.790749337Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 297.862206ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.804796199Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 860.700389ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.818633827Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 914.660657ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:10.95525786Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.255970183s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.027626645Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 1.073331655s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.061366347Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 548.942711ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.074164523Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 329.540523ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.114701477Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 881.507601ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.125870809Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 624.614174ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.136523351Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 1.022325299s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.282542258Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 513.093397ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.357808143Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 1.050707225s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.369308333Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 709.201981ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.457170941Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 1.347231978s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.613120025Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 776.032126ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.635531801Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 1.234694938s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.66335005Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 629.795621ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.693703012Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 1.041026489s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.730006567Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 643.569305ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.772598123Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 1.110329784s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.834110893Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 738.081824ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.88153469Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 586.745915ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.893072279Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 1.328108368s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.916989367Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 616.691972ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:11.963664028Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 1.378920914s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.019415348Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 1.062282019s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.029721582Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 589.245503ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.152765493Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 1.415786126s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.17909328Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.013116895s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.297984153Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 539.704119ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.373973214Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 901.398021ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.385842451Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 1.033857451s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.436986655Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 1.082133849s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.447947663Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 1.385040595s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.458625955Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 603.732884ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.492315636Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 546.905954ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.596624936Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 798.19698ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.640187452Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Profile for oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"oxide/zone-network-setup\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlService23\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"static_addr\" value=\"fd00:1122:3344:101::e\"/>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/ssh\">\n    <instance enabled=\"false\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"oxide/clickhouse\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::e\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"8123\"/>\n        <propval type=\"astring\" name=\"store\" value=\"/data\"/>\n      </property_group>\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/install\">\n      <property_group type=\"application\" name=\"install_props\">\n        <property type=\"net_address\" name=\"nameserver\">\n          <net_address_list>\n            <value_node value=\"fd00:1122:3344:3::1\" />\n            <value_node value=\"fd00:1122:3344:1::1\" />\n            <value_node value=\"fd00:1122:3344:2::1\" />\n          </net_address_list>\n        </property>\n      </property_group>\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n  <service version=\"1\" type=\"service\" name=\"network/dns/client\">\n    <instance enabled=\"true\" name=\"default\">\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:12.652328535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:12.653034909Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 951.509426ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.756752337Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 746.175243ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.784607947Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 58.561598ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.89962086Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 708.802853ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.91403905Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 1.432240083s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.927457149Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 718.649724ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.959099967Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 116.051063ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.971050966Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 696.004218ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:12.985846624Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 111.495997ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.122538908Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 1.186057807s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.164271886Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 1.15870666s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.207282759Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 802.699278ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.2221895Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 504.484904ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.332194088Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.341261827Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:426","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"oximeter/producer/src/lib.rs:426","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/metrics/producers): operation timed out\"","delay":"86.809086607s"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 1.423001353s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.349521159Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 658.339003ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.363950809Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.371182126s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.376337407Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 1.150556605s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.411127111Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 1.149615517s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.452039922Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 631.371466ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.480934813Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 1.403064319s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.540779038Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 920.807947ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.551035961Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 1.058413597s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.605594446Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 864.159365ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.633410385Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 1.277260488s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.689305348Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 1.161489836s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.700124683Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 558.418778ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.719070117Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 1.053623966s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.849043011Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 1.308004939s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:13.863918931Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 1.179343084s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.039527824Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 845.634618ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.055378934Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 991.24235ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.127557086Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 906.538841ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.304347583Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 1.152650042s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.377573337Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 629.000463ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.400211888Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 1.40248341s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.428585918Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 1.464903022s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.494151132Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 643.901069ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.53825616Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 1.090909125s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.584564164Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 1.043662349s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.617861816Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 1.008319442s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.682982451Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.282779159s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.766246283Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 636.933288ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.796533263Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 1.022063988s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.880038771Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 1.150423272s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.922818701Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 1.453159607s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.94247902Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 708.472654ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.962510636Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 759.783325ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:14.984425243Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 528.256604ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.04485008Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 1.417235023s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.133527634Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 990.520707ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.19199482Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 1.472801124s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.205498751Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 797.24176ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.222830282Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 1.496639761s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.236015866Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 1.353067187s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.450400806Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 1.159405902s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.543719767Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 915.292997ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.590779966Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 734.952931ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.675774515Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 844.183427ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.689144743Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 1.439051992s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.70390041Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 1.497972851s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.718497614Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 1.427882038s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.756459653Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 1.331791726s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.841538294Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607\") failed: Property not found. retry in 1.304453859s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.916778629Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 991.347004ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:15.97401774Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 1.437009393s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.031932225Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.428543717s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.059375916Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 615.056511ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.088274227Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a\") failed: Property not found. retry in 1.216011795s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.197132201Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 643.465493ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.409677103Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 977.222651ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.421970729Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8\") failed: Property not found. retry in 706.421811ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.520130591Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 818.545068ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.545661132Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 1.045946717s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.569288983Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 1.102909961s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.69120785Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b\") failed: Property not found. retry in 614.670761ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.71571479Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 889.513972ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.727120977Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d\") failed: Property not found. retry in 828.220259ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.745266685Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 948.520622ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.816791503Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 582.662369ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:16.977232131Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 943.743124ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.067601851Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 1.032182523s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.158453441Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 1.036847359s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.190397596Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 613.853296ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.202009437Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 1.164717044s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.22859985Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 1.442480685s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.379481079Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 584.773164ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.413133429Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 1.248568006s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.481312258Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 629.774355ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.499387974Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 642.826381ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.571628237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 846.772815ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.628216354Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 543.4799ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.640171563Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 1.391797863s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.780581504Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 771.575832ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.80969297Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 503.859158ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:17.831116346Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054\") failed: Property not found. retry in 1.287753794s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.010903896Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 1.189568454s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.034501247Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 675.214636ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.14665768Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e\") failed: Property not found. retry in 873.506177ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.196598938Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 518.999038ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.207542516Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 502.924566ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.228267237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8\") failed: Property not found. retry in 761.042156ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.25005386Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 1.487241809s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.352858179Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 1.449707724s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.406599487Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 1.279970867s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.492049135Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b\") failed: Property not found. retry in 741.054291ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.595178301Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 979.168261ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.743931945Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 1.218236171s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.761403844Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288\") failed: Property not found. retry in 995.316115ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.775238197Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc\") failed: Property not found. retry in 697.991814ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.837061023Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550\") failed: Property not found. retry in 1.24416045s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:18.85085933Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 779.848844ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.189185919Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 1.006240209s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.242279803Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 723.503383ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.739456026Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 1.446870095s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.807642305Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 504.748212ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.85208833Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 747.188043ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.868363078Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 1.092358054s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.981724556Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 718.369455ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:19.993108703Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 1.399675741s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:20.260766911Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 872.868086ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:20.368907861Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e\") failed: Property not found. retry in 1.319788345s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:20.473223921Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\") failed: Property not found. retry in 1.225429317s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:20.631502864Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f\") failed: Property not found. retry in 1.141244141s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:20.725690284Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834\") failed: Property not found. retry in 1.306120121s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:21.086165363Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25\") failed: Property not found. retry in 536.034198ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:21.252058234Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959\") failed: Property not found. retry in 817.142215ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:21.264530743Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05\") failed: Property not found. retry in 1.088846718s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:21.678137719Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811503298Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_e5538333-fcad-480c-9181-aa7e94aaf834"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811553449Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_external_dns_12908044-fccf-4c46-8e7c-c2a0be1bfb25"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811563579Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_oximeter_7d8b5024-88b0-4173-8867-4e833f05550e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811569869Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_638e5dab-76c4-4b78-b15a-a7faf399d1d8"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811575569Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_7b766ca5-5d0c-459d-bc67-f339bcac986b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.81158042Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_03e9085d-76ce-4798-9d35-12ceecd4c607"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.81158528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d62b80e0-a2d7-4b05-8ee7-7f6a9516881e"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.81159035Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_99861af6-438d-4bef-ba93-632645b46d2f"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_9cebcc66-fa56-4e34-af63-2c7ac52ff959"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.81160018Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_ae891598-60c4-47ca-bd0b-302a9e60a0bc"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811638341Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_42687f02-f9e0-4e2e-8b1e-8fcc2648d21d"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.811797064Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_d41b4327-a68c-4bde-a48b-cfcb75b4c054"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.813102681Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_eece7197-e5eb-4531-85f7-6700f07faf6b"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.813242474Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_c06cd1db-bb07-41fa-af89-0fd52ff0a0a8"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.813254204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_pantry_d2e82ff1-e608-4c9b-8b24-06c6a00e8c2a"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.813285595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_ecac0828-7786-4503-af9f-835e69870288"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.813291855Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_crucible_e0bae9f0-075a-41e1-a5e1-22133ae05550"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.814240925Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_clickhouse_7ce930b4-1bea-41ce-8c9a-2aa16fd16d05"}
{"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.814256425Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3026","zone":"oxz_nexus_feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.823735113Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.825935028Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.834638929Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c072f072-967b-4315-9f5d-017448974204","remote_addr":"[fd00:1122:3344:101::1]:60075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":24698311,"response_code":"204"}
{"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.867985113Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1236"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.868256349Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.868718798Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"}
{"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.881046975Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:680"}
{"msg":"Nexus address: [fd00:1122:3344:101::a]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.894232289Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:693"}
{"msg":"rack_network_config: RackNetworkConfigV1 {\n    bfd: [],\n    bgp: [],\n    infra_ip_first: 10.151.2.174,\n    infra_ip_last: 10.151.2.174,\n    ports: [\n        PortConfigV1 {\n            addresses: [\n                V4(\n                    Ipv4Network {\n                        addr: 10.151.2.174,\n                        prefix: 24,\n                    },\n                ),\n            ],\n            autoneg: false,\n            bgp_peers: [],\n            port: \"qsfp0\",\n            routes: [\n                RouteConfig {\n                    destination: V4(\n                        Ipv4Network {\n                            addr: 0.0.0.0,\n                            prefix: 0,\n                        },\n                    ),\n                    nexthop: 10.151.2.100,\n                    vlan_id: None,\n                },\n            ],\n            switch: Switch0,\n            uplink_port_fec: None,\n            uplink_port_speed: Speed40G,\n        },\n    ],\n    rack_subnet: Ipv6Network {\n        addr: fd00:1122:3344:100::,\n        prefix: 56,\n    },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:22.974792125Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:817"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:23.307097678Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:23.314590394Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:23.464436931Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:23.550739017Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:23.627043585Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:23.807410697Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:23.91666794Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:24.192978258Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:24.496331619Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:24.969727188Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:25.075220383Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:25.18853506Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:25.560458197Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:25.942845933Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:26.991376548Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:27.075400396Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:28.106203621Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:29.070311389Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:12:29.095591255Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:29.438977288Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:12:30.098387047Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::a]:12221/racks/89a4c48a-3dff-478b-98cb-9f441a913804/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:30.878967077Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:12:31.066630611Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:12:31.10169661Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/acb2a796-8e9f-4811-b601-726a12d79b0b): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-05-15T07:12:32.104906551Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"}
{"msg":"Failed to handoff to nexus: Error Response: status: 403 Forbidden; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3d298a7a-a40d-470b-b20b-940db826d743\", \"content-length\": \"113\", \"date\": \"Wed, 15 May 2024 07:12:32 GMT\"}; value: Error { error_code: Some(\"Forbidden\"), message: \"Forbidden\", request_id: \"3d298a7a-a40d-470b-b20b-940db826d743\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:32.894948348Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:33.526868385Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59637"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:33.527739112Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5c42073e-5377-4592-9def-5e72c6986bd7","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59637","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":237,"response_code":"200"}
{"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b79cbd22-932d-4a9d-a4c4-c77c63bd79f5\", \"content-length\": \"173\", \"date\": \"Wed, 15 May 2024 07:12:34 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"acb2a796-8e9f-4811-b601-726a12d79b0b\\\"\", request_id: \"b79cbd22-932d-4a9d-a4c4-c77c63bd79f5\" }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:34.987762578Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:879"}
{"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.13161676Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.257895697Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37863"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.259063161Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.352337262Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"8bb9e1e4-828b-480f-8dde-3edab2b98673","remote_addr":"[fd00:1122:3344:101::a]:37863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":94159,"response_code":"200"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.354243951Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.354907445Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.356444677Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:35.356894736Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"205076cb-ce30-4ac6-8e07-166edaceb5a8","remote_addr":"[fd00:1122:3344:101::a]:37863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":4215,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.779776247Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61404"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.78282715Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.782871511Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.782915432Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.782948073Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.782961703Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.782967553Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.782972633Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.782983354Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"02e530e1-39fc-4701-a212-3b271b8e3b73","remote_addr":"[fd00:1122:3344:101::a]:61404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3180,"response_code":"204"}
{"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.785144028Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:889"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.934140828Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57963"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:38.934252141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.041929961Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"9d8dc2ca-ff05-41df-a8e2-ba3fe70befbb","remote_addr":"[fd00:1122:3344:101::a]:57963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":107667,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042052484Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43216"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042311049Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Reading ledger from /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042415161Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042481702Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042540384Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042616755Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042645086Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042674457Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042693707Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042724438Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.042743308Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9ad20360-890f-488d-ae51-0f889648f461","remote_addr":"[fd00:1122:3344:101::a]:43216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":592,"response_code":"204"}
{"msg":"Reading ledger from /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.04285404Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.043139466Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"1a0a8a9d-6087-4119-9f1a-9929afb0490f","remote_addr":"[fd00:1122:3344:101::a]:57963","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1009,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.0625744Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36449"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.063467459Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.0635124Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.063560131Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.063571771Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.063586292Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.063600272Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.063609622Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.063634962Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ba17ed3d-9f39-445c-830a-acdde3c7bbf1","remote_addr":"[fd00:1122:3344:101::c]:36449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":899,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119096296Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35769"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119353372Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119405523Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119438443Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119446764Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119465134Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119476404Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119507415Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:12:39.119523515Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3d7e6a9a-1e6a-41a5-a4f0-fb0ddb8a65c7","remote_addr":"[fd00:1122:3344:101::b]:35769","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":317,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:02.948599686Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52531"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.196875341Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"28bc9a69-71bd-4fc2-8bac-5108a2ff243b","remote_addr":"[fd00:1122:3344:101::a]:52531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":248208,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.197093956Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49544"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.197181778Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43840"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.207792928Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9f4826e5-c9ca-4732-ae85-483f78b1ea79","remote_addr":"[fd00:1122:3344:101::a]:52531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":10586,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.264765633Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e0aee942-eb3e-439a-880e-b79223f15578","remote_addr":"[fd00:1122:3344:101::c]:43840","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":67436,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.268494611Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8299ffd6-6b63-4d3c-9639-696ca9d6d066","remote_addr":"[fd00:1122:3344:101::c]:43840","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":667,"response_code":"200"}
{"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.302914417Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.310677669Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.315340646Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.316131112Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.316160593Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"}
{"msg":"Writing ledger to /pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.316210114Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.316358617Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"7b200b6c-15f2-4e13-b9f5-9b9bf79a627d","remote_addr":"[fd00:1122:3344:101::a]:52531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":26688,"response_code":"204"}
{"msg":"Writing ledger to /pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.316442789Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"}
{"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.316584012Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.316654773Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"f437cb59-2aa6-433a-9663-0f1b24219639","remote_addr":"[fd00:1122:3344:101::c]:43840","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":13480,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.323571117Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ef35cae4-c56a-49ee-92bf-a967c27388a5","remote_addr":"[fd00:1122:3344:101::b]:49544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":126368,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.323970085Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"05a1649b-612e-4a55-b5a8-8f7d34ece348","remote_addr":"[fd00:1122:3344:101::b]:49544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.560480886Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45410"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:03.560732551Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"233b3317-ec58-479a-869f-a47c4c40cae9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45410","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":36,"response_code":"200"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:20.838386933Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.135478385Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44762"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.185601537Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"398cfca2-28fc-4e98-a923-0c771c19a7cb","remote_addr":"[fd00:1122:3344:101::a]:44762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":50064,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.18573712Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35727"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.189165901Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f865349d-bf7a-4c34-ab14-2300ce7b3dce","remote_addr":"[fd00:1122:3344:101::a]:44762","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3185,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.202606741Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55652"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.235821701Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8ca03e7a-ac82-483d-a49b-ed393364ed78","remote_addr":"[fd00:1122:3344:101::c]:35727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49954,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.239676212Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"00fc9f5f-9aa9-45e7-85c5-8fe7afc30c4f","remote_addr":"[fd00:1122:3344:101::c]:35727","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":986,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.282038133Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7b7de0ae-6d0c-4501-a248-62740839f63f","remote_addr":"[fd00:1122:3344:101::b]:55652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":79262,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.28237391Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4a221d60-f3ec-4e3a-9fd9-8a6de2c39908","remote_addr":"[fd00:1122:3344:101::b]:55652","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.594683347Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34286"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:13:33.59478916Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b4981772-3d6f-49c7-b1d8-9bf0a075ec30","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34286","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:02.9098313Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60996"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:02.959307119Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"beddff37-a382-4d61-904f-aba722256fe0","remote_addr":"[fd00:1122:3344:101::c]:60996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49424,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:02.959754889Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3f578d53-8574-46fa-9aa8-d03c564af5ea","remote_addr":"[fd00:1122:3344:101::c]:60996","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":210,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:02.968027201Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35320"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:03.015441157Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"da7002e7-726e-4a6f-9c72-d82f4f76d353","remote_addr":"[fd00:1122:3344:101::a]:35320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":47309,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:03.01559213Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64339"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:03.019586273Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b614e5d8-1a22-48a1-8c33-713d98f1ffed","remote_addr":"[fd00:1122:3344:101::a]:35320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3709,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:03.06217802Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"daff3b85-32b6-4627-ac8b-a61a917fee18","remote_addr":"[fd00:1122:3344:101::b]:64339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46405,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:03.062542397Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b1045de8-73c2-48e8-947a-5e584d0505b4","remote_addr":"[fd00:1122:3344:101::b]:64339","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":167,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:03.628340868Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44913"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:03.62941181Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"04b63b70-1d66-42ed-8a18-0b6c424f9eed","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44913","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":748,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:10.263526148Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:65149"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:10.271378492Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"05e61014-d288-48f3-abc1-40686d810572","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":6994,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.179186042Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63438"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.22860839Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b2332f70-30d8-4049-9ff6-a90999c57f69","remote_addr":"[fd00:1122:3344:101::a]:63438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49360,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.228894806Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b6a6d308-ebad-40ab-b786-febd7cd991e2","remote_addr":"[fd00:1122:3344:101::a]:63438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.310150886Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52997"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.359661707Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1028b17c-24b6-4f71-8afb-957a02735929","remote_addr":"[fd00:1122:3344:101::b]:52997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49443,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.359914012Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35614"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.360118116Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"5efed80f-370e-4a01-b84c-10560607c20a","remote_addr":"[fd00:1122:3344:101::b]:52997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":213,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.407551143Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c132f461-b434-4b96-b0ca-c93b8869a117","remote_addr":"[fd00:1122:3344:101::c]:35614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":47495,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.407920691Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"59e9806c-713d-4ed5-b8b7-57552c509996","remote_addr":"[fd00:1122:3344:101::c]:35614","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":147,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.660928435Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48814"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:33.661050737Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1fc36acd-4f1d-4592-93c1-03aa46db3acb","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48814","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:14:40.26269186Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"6c4a0b17-7b19-4008-8484-8da5257a493b","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1233,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:02.981804246Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60589"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.041330445Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"36449bb0-3ac6-4f3e-aa44-54a7b930ca8e","remote_addr":"[fd00:1122:3344:101::b]:60589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":59481,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.041483738Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35240"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.08966739Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"80c893df-17ee-488d-bd0e-5dfc8ff37d63","remote_addr":"[fd00:1122:3344:101::c]:35240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48105,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.089895185Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0f9b5950-51f3-4004-a26a-75fe609dbf60","remote_addr":"[fd00:1122:3344:101::b]:60589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.090779424Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6d67ada2-1e64-45ad-9edd-9f5da103713d","remote_addr":"[fd00:1122:3344:101::c]:35240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":791,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.107735406Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62753"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.156538852Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"964f7f9d-0b46-4039-a788-9acb3ffd61c5","remote_addr":"[fd00:1122:3344:101::a]:62753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48741,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.156811457Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7f82a5fe-dd7a-4401-8325-2324b157352e","remote_addr":"[fd00:1122:3344:101::a]:62753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.694839591Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51655"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:03.694922233Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"12776a45-b514-45db-a893-f7b27ff570ef","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51655","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":78,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:10.262022247Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"dec04add-b9ac-4431-ab87-3dd1cfb7340e","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":775,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.101772434Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54570"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.152937119Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3bc4c5cb-489e-4fb5-9c7a-c6ab1063fc22","remote_addr":"[fd00:1122:3344:101::c]:54570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":51104,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.153067221Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33472"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.156652346Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"aa61d6bc-2b3e-4d02-9842-4a2a94bf2b25","remote_addr":"[fd00:1122:3344:101::c]:54570","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3361,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.199529027Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8a27fe0f-1913-4d4a-b951-c1a1e59e9544","remote_addr":"[fd00:1122:3344:101::a]:33472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46362,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.200861285Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d3e9e866-417a-41ae-9ace-e013b1e270b5","remote_addr":"[fd00:1122:3344:101::a]:33472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1108,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.243553893Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:58717"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.290155703Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c99d36af-89c0-46b0-8dcc-ec616b368cfa","remote_addr":"[fd00:1122:3344:101::b]:58717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46535,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.290405228Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"093c0b29-54eb-4157-954a-72e6f664193c","remote_addr":"[fd00:1122:3344:101::b]:58717","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.728579534Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37563"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:33.728713657Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8ad307c4-0365-4827-b8af-9311d6d0b03b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37563","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":77,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:15:40.261319124Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"399eac94-8b08-466d-9316-7c24e014a305","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":591,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:02.90812055Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34134"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:02.958151331Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b88c276e-68d2-4285-9b07-3f559b8d14dc","remote_addr":"[fd00:1122:3344:101::c]:34134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49974,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:02.958421897Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"da56b1a8-3983-4363-b7c1-b59de8d17585","remote_addr":"[fd00:1122:3344:101::c]:34134","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:02.984618962Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:42671"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:03.031622209Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"dcd2c5fb-0918-4af0-88b2-860a9228f29c","remote_addr":"[fd00:1122:3344:101::a]:42671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46892,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:03.031918346Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b27630f0-7534-4c06-b11d-c6ead4a898c1","remote_addr":"[fd00:1122:3344:101::a]:42671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:03.07535521Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56470"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:03.12200809Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ea07257b-e0ce-4abe-a469-6c9d36ca3665","remote_addr":"[fd00:1122:3344:101::b]:56470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46565,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:03.122271126Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"95ca3787-0fc2-495c-a824-675f651e3d4e","remote_addr":"[fd00:1122:3344:101::b]:56470","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:03.769729585Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61240"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:03.769845888Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6ef44f29-34d2-4950-80f5-2b577cc1aa84","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61240","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:10.274069154Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"03d350f2-1166-4f34-a506-659ce5186e17","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":884,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.118854558Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59864"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.168335318Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ef700789-750d-4890-909d-faec180e2044","remote_addr":"[fd00:1122:3344:101::c]:59864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49414,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.168588793Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"75d82a7e-a12b-4d4a-b95e-11dd8637b425","remote_addr":"[fd00:1122:3344:101::c]:59864","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.175908685Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37882"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.225130459Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ceeff066-b412-41de-9722-a6d6bc38e5b2","remote_addr":"[fd00:1122:3344:101::a]:37882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49116,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.225278102Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44747"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.228831105Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3b130e9e-97e5-4cbb-8762-514d34343a0c","remote_addr":"[fd00:1122:3344:101::a]:37882","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3334,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.271586645Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8f77a1dd-6d51-4a50-b3ad-73f2001f9d06","remote_addr":"[fd00:1122:3344:101::b]:44747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46224,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.271934182Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"82ec753d-08ba-49a6-8f2d-c5930aef8540","remote_addr":"[fd00:1122:3344:101::b]:44747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":137,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.801637423Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47426"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:33.801729385Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e26513a4-0a72-4b78-9897-bcfd91e5fb3f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47426","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":69,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:16:40.310897734Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"33088b5c-34c2-4641-b528-cb827b6fe05f","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":735,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:02.915652724Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36935"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:02.966030632Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5bca168d-11fb-4e93-bc06-e656ea964358","remote_addr":"[fd00:1122:3344:101::a]:36935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":50315,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:02.966156835Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56807"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:02.970376373Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"994624f3-5f7e-4674-bdb7-94bd04721d04","remote_addr":"[fd00:1122:3344:101::a]:36935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":3963,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:03.015505992Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"addcbcc2-6626-4519-a575-0103b6112dbf","remote_addr":"[fd00:1122:3344:101::b]:56807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49240,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:03.015878209Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"156b7690-1207-4547-b988-2b559a85a53a","remote_addr":"[fd00:1122:3344:101::b]:56807","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":133,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:03.089584693Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33083"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:03.134322564Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c8eaabf7-6951-4f6b-b89b-1083afd1af9f","remote_addr":"[fd00:1122:3344:101::c]:33083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":44669,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:03.134594909Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"83221fb5-99b6-4fc0-ad1f-75385bd81229","remote_addr":"[fd00:1122:3344:101::c]:33083","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":106,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:03.832647561Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44995"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:03.832740273Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"894d0526-cf1f-4676-8933-ad4c04fffc57","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44995","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:10.310741074Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"a97ae30f-2b33-4f80-be98-ef985a0304bc","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":620,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.04564525Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:45132"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.045890025Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.045946416Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.045984097Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.045998527Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.046006427Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.046012257Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.046028618Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.046055248Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c116ff48-a53b-4574-80ca-116b0fb617e1","remote_addr":"[fd00:1122:3344:101::c]:45132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":377,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.101801658Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60191"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.149918579Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d5e06d5a-dbc6-4193-be79-ddecfb4b0aef","remote_addr":"[fd00:1122:3344:101::c]:60191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48081,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150044771Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56776"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150362968Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0d55b02d-f9d3-4c1e-91c3-2ccd1f44e1f1","remote_addr":"[fd00:1122:3344:101::c]:60191","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":187,"response_code":"200"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150388789Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.15044015Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150504191Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150531342Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150564402Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150584393Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150598763Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.150624384Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"a88ed11f-061b-4ebc-aac1-8ebcbb4ecca3","remote_addr":"[fd00:1122:3344:101::b]:56776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":496,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.169617279Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34152"}
{"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.169926295Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:525","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 2e30c7c4-8de6-4cbd-8a3d-052bb77d3985, kind: Service { id: e5538333-fcad-480c-9181-aa7e94aaf834 }, name: Name(\"external-dns-e5538333-fcad-480c-9181-aa7e94aaf834\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 204, 142])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: fe506ce4-c84e-4ef5-9ed5-5c96a847dd43, kind: Service { id: 12908044-fccf-4c46-8e7c-c2a0be1bfb25 }, name: Name(\"external-dns-12908044-fccf-4c46-8e7c-c2a0be1bfb25\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 254, 70])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.1.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38eee15b-cb30-4e2c-b5c3-f5511aac6743, kind: Service { id: 99861af6-438d-4bef-ba93-632645b46d2f }, name: Name(\"nexus-99861af6-438d-4bef-ba93-632645b46d2f\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 222, 14])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 88dead63-2254-4036-85c5-6efbd3602a0d, kind: Service { id: feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3 }, name: Name(\"nexus-feeb24cb-4b3b-4c4b-b3ef-d86f60b83ea3\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 207, 1])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 9b63bcc5-cf23-47dc-87dc-38f5ac4bcfd6, kind: Service { id: 9cebcc66-fa56-4e34-af63-2c7ac52ff959 }, name: Name(\"nexus-9cebcc66-fa56-4e34-af63-2c7ac52ff959\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 234, 140])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.2.0, prefix: 24 })), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.169991006Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.170029757Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.170042348Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.170060238Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.170077848Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[]","port":"\"opte0\""}
{"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.170090789Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:544","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.170115279Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"59663957-072b-4778-9039-c175fd8162c0","remote_addr":"[fd00:1122:3344:101::a]:34152","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":406,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.229771249Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:58296"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.279106056Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"27187d12-4698-475e-8a1e-3b035f150a53","remote_addr":"[fd00:1122:3344:101::b]:58296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49289,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.279386932Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f8acf92b-29c1-473e-8659-7ccbbc752d1a","remote_addr":"[fd00:1122:3344:101::b]:58296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.294130088Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33458"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.339485412Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6df35e5d-75ff-4de8-84f3-0f54314c672e","remote_addr":"[fd00:1122:3344:101::a]:33458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":45250,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.339939601Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1d4481a5-ac6c-449c-aacf-2cb7263fb11d","remote_addr":"[fd00:1122:3344:101::a]:33458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":203,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.865572007Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64069"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:33.86568912Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"bbca4da6-1609-44b9-8c76-a5bbc05b22a4","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64069","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:17:40.310273625Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"5f1cf87e-c336-482a-8e99-c26151e53f60","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":788,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:02.905555954Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51220"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:02.955472603Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bcc55ac7-fec5-496b-96aa-81fb815089a9","remote_addr":"[fd00:1122:3344:101::c]:51220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":49848,"response_code":"204"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:02.955879951Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55699"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:02.955951253Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38710"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:02.961513138Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"31d894db-2652-408b-97cd-ca3f2eec2ba4","remote_addr":"[fd00:1122:3344:101::c]:51220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":5695,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:03.004919292Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"4f7f0d85-b9b5-49b9-85b8-fad07cc24570","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48873,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:03.009579489Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a3e8aab2-6689-4066-a1de-434396b22abe","remote_addr":"[fd00:1122:3344:101::a]:55699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1341,"response_code":"200"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:03.051095852Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1ef7ee96-d501-4cd0-8955-ab386abbeddd","remote_addr":"[fd00:1122:3344:101::b]:38710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":94970,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:03.051445269Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"729c1575-4c82-4072-949f-a5459e45c444","remote_addr":"[fd00:1122:3344:101::b]:38710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:03.899016573Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53753"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:03.899117095Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"3a98d705-2216-4d0c-95e9-839e0d56bd71","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53753","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":62,"response_code":"200"}
{"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:07.237832575Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:07.238246273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:07.276122021Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999997091s","next_cleanup":"Instant { tv_sec: 1851, tv_nsec: 684877306 }"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:10.309984938Z","hostname":"sock","pid":1555,"uri":"//acb2a796-8e9f-4811-b601-726a12d79b0b","method":"GET","req_id":"b92bca1d-92c7-4f12-b3c8-084def773135","remote_addr":"[fd00:1122:3344:101::d]:65149","local_addr":"[fd00:1122:3344:101::1]:55688","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"acb2a796-8e9f-4811-b601-726a12d79b0b","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":456,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.269425506Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:32773"}
{"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.2792186Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:546","metadata":"InstanceMetadata { silo_id: 82ce2851-ee69-4933-b9a7-f0eee8dffe12, project_id: 22d38dba-1ffb-40fc-b3f3-4ee2a548bcf6 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-15T07:18:16.763717Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(12006808))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-7d72237e3cb\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, read_only_parent: Some(Volume { block_size: 512, id: 064e6874-7451-4713-8b9b-16ba9758e09b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 98f450bf-ba2d-4a97-b47f-1d65c4edbb9e, key: Some(\"N/PJy4twLSmZ7nLWEfFW8oifHAzQ1XQV3m2X/kcGn88=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, key: Some(\"08OcIC7oIXvo3LKwtBjCBiI0Wxg52LMrRfXsu93NGs8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::13]:19002\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB6SDqvWK9YAABIOq9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAe0g6r1ivWAAASDqvWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NGI1ZDkzZS0yMjQxLTQzMTktODEyOS1mZTY3MTA4YzhmNDkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDlyZFRFMkR3WXU5WGloTWxwdDQ4ZzU3OUhjZkROMDQ0MXpKTVhIbUFUZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.279368343Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:580","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.279747911Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:938","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(12006808))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 6a12a6a1-9861-4fe8-bb03-f96ecb313186, kind: Instance { id: 74b5d93e-2241-4319-8129-fe67108c8f49 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 241, 86, 179])), subnet: V4(Ipv4Net(Ipv4Network { addr: 172.30.0.0, prefix: 22 })), vni: Vni(12006808), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-7d72237e3cb\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, read_only_parent: Some(Volume { block_size: 512, id: 064e6874-7451-4713-8b9b-16ba9758e09b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 98f450bf-ba2d-4a97-b47f-1d65c4edbb9e, key: Some(\"N/PJy4twLSmZ7nLWEfFW8oifHAzQ1XQV3m2X/kcGn88=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, key: Some(\"08OcIC7oIXvo3LKwtBjCBiI0Wxg52LMrRfXsu93NGs8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::13]:19002\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB6SDqvWK9YAABIOq9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAe0g6r1ivWAAASDqvWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NGI1ZDkzZS0yMjQxLTQzMTktODEyOS1mZTY3MTA4YzhmNDkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDlyZFRFMkR3WXU5WGloTWxwdDQ4ZzU3OUhjZkROMDQ0MXpKTVhIbUFUZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-05-15T07:18:16.763717Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.28262746Z","hostname":"sock","pid":1555,"uri":"/instances/74b5d93e-2241-4319-8129-fe67108c8f49","method":"PUT","req_id":"1ab5419d-8210-446d-b805-9ef2d6bffd63","remote_addr":"[fd00:1122:3344:101::a]:32773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":13118,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.397230395Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55817"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.397703035Z","hostname":"sock","pid":1555,"uri":"/instances/74b5d93e-2241-4319-8129-fe67108c8f49/state","method":"PUT","req_id":"a6cd5a60-9841-4bd2-95e5-e097ae602b03","remote_addr":"[fd00:1122:3344:101::a]:55817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":392,"response_code":"200"}
{"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.422735346Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:394","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 241, 86, 179]), slot: 0, vni: Vni { inner: 12006808 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.423561743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Configuring new Omicron zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.459664364Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"}
{"msg":"Installing Omicron zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:17.480568479Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"}
{"msg":"Profile for oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543:\n<!DOCTYPE service_bundle SYSTEM \"/usr/share/lib/xml/dtd/service_bundle.dtd.1\">\n<service_bundle type=\"profile\" name=\"omicron\">\n  <service version=\"1\" type=\"service\" name=\"system/illumos/propolis-server\">\n    <instance enabled=\"true\" name=\"default\">\n      <property_group type=\"application\" name=\"config\">\n        <propval type=\"astring\" name=\"datalink\" value=\"oxControlInstance0\"/>\n        <propval type=\"astring\" name=\"gateway\" value=\"fd00:1122:3344:101::1\"/>\n        <propval type=\"astring\" name=\"listen_addr\" value=\"fd00:1122:3344:101::1:0\"/>\n        <propval type=\"astring\" name=\"listen_port\" value=\"12400\"/>\n        <propval type=\"astring\" name=\"metric_addr\" value=\"[fd00:1122:3344:101::c]:12221\"/>\n      </property_group>\n    </instance>\n  </service>\n</service_bundle>","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:18.856335712Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"}
{"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:18.856549206Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 59.142966ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:19.017893573Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 59.936349ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:19.092206819Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 113.110852ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:19.169853214Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 577.037868ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:19.300599554Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 1.010831071s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:19.888241459Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:20.837348074Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 1.036408746s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:20.909582248Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 796.304506ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:21.958694263Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 715.342924ms","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:22.767425639Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"wait for service Some(\"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\") failed: Property not found. retry in 1.320581035s","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:23.49330225Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"}
{"msg":"Started propolis in zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:24.841769795Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:1423"}
{"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:24.849826392Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:1432"}
{"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:24.884686248Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:1446"}
{"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAB6SDqvWK9YAABIOq9YAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAe0g6r1ivWAAASDqvWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI3NGI1ZDkzZS0yMjQxLTQzMTktODEyOS1mZTY3MTA4YzhmNDkiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTDlyZFRFMkR3WXU5WGloTWxwdDQ4ZzU3OUhjZkROMDQ0MXpKTVhIbUFUZSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-7d72237e3cb\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, read_only_parent: Some(Volume { block_size: 512, id: 064e6874-7451-4713-8b9b-16ba9758e09b, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 98f450bf-ba2d-4a97-b47f-1d65c4edbb9e, key: Some(\"N/PJy4twLSmZ7nLWEfFW8oifHAzQ1XQV3m2X/kcGn88=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::19]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: a2bdf7c7-d54c-4168-8d09-daf3c96c27c1, key: Some(\"08OcIC7oIXvo3LKwtBjCBiI0Wxg52LMrRfXsu93NGs8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::16]:19001\", \"[fd00:1122:3344:101::19]:19002\", \"[fd00:1122:3344:101::13]:19002\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 74b5d93e-2241-4319-8129-fe67108c8f49, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 22d38dba-1ffb-40fc-b3f3-4ee2a548bcf6, silo_id: 82ce2851-ee69-4933-b9a7-f0eee8dffe12 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:24.886956845Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:685"}
{"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:25.06793034Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:687"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-05-15T07:18:25.069060454Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:25.06935757Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:25.0693753Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-15T07:18:25.069060454Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }","propolis_id":"b926d0d2-78b1-4d16-b1ed-d24aaeaf7543"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:25.069405821Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-05-15T07:18:25.069060454Z } }","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-05-15T07:18:25.127457469Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:25.127483329Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:25.12750845Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-15T07:18:25.127457469Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }","propolis_id":"b926d0d2-78b1-4d16-b1ed-d24aaeaf7543"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:25.12751874Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-05-15T07:18:25.127457469Z } }","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:32.933920338Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50267"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:32.934164213Z","hostname":"sock","pid":1555,"uri":"/instances/74b5d93e-2241-4319-8129-fe67108c8f49/state","method":"GET","req_id":"40203884-97a7-466f-980a-3136923c76ce","remote_addr":"[fd00:1122:3344:101::c]:50267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":204,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.028991526Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64479"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.02919981Z","hostname":"sock","pid":1555,"uri":"/instances/74b5d93e-2241-4319-8129-fe67108c8f49/state","method":"GET","req_id":"ebc4ba71-5f07-419a-aebf-af82ff730189","remote_addr":"[fd00:1122:3344:101::a]:64479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":151,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.173173206Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51655"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.173281678Z","hostname":"sock","pid":1555,"uri":"/instances/74b5d93e-2241-4319-8129-fe67108c8f49/state","method":"GET","req_id":"d8b25a24-acdc-42bf-8d59-e248dd96d12c","remote_addr":"[fd00:1122:3344:101::b]:51655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":72,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.175038664Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36263"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.223589243Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"963148da-eb4b-4365-be4c-4f5dc58fae0f","remote_addr":"[fd00:1122:3344:101::c]:36263","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":48465,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.22439157Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7a5512d1-ee04-4237-9827-4104e81b6087","remote_addr":"[fd00:1122:3344:101::c]:36263","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":638,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.266623829Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62458"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.31427252Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f5eb8092-9b65-4afd-9394-6f6b5f0fe954","remote_addr":"[fd00:1122:3344:101::a]:62458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":47563,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.314529935Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2da0135a-afee-4331-beea-1f0e2e92ee83","remote_addr":"[fd00:1122:3344:101::a]:62458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.323631775Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42679"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.370445659Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1111ecf7-a186-4f17-a519-f0387065f518","remote_addr":"[fd00:1122:3344:101::b]:42679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":46702,"response_code":"204"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.370743295Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3ef5cddd-cc5d-4746-a887-3d8760409416","remote_addr":"[fd00:1122:3344:101::b]:42679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.933166316Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55095"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:33.933243428Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"fb108a3e-8f3c-4e3d-aed5-7d261f118d40","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55095","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":71,"response_code":"200"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.845087478Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59545"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.845718741Z","hostname":"sock","pid":1555,"uri":"/instances/74b5d93e-2241-4319-8129-fe67108c8f49/state","method":"PUT","req_id":"b9df7b19-4860-482a-b149-1f31e8f09abd","remote_addr":"[fd00:1122:3344:101::a]:59545","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":606,"response_code":"200"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-05-15T07:18:34.845824263Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.845849204Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.845861854Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-15T07:18:34.845824263Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }","propolis_id":"b926d0d2-78b1-4d16-b1ed-d24aaeaf7543"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.845891035Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(b926d0d2-78b1-4d16-b1ed-d24aaeaf7543), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-05-15T07:18:16.811647382Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-05-15T07:18:34.845824263Z } }","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-05-15T07:18:34.891896712Z }","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.891924163Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:569"}
{"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.891950343Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:591","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-05-15T07:18:34.891896712Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-15T07:18:34.891896712Z }","propolis_id":"b926d0d2-78b1-4d16-b1ed-d24aaeaf7543"}
{"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.891970264Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.891994474Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.892207878Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"}
{"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:34.89226799Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/3907cd6d-3c6d-4543-92de-9ec01f9bc32c/debug/bundle/zone\", \"/pool/int/c3ed08c3-e0e1-43f2-b040-3c4755a2167c/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/4e6ef30f-a7d9-4acc-99aa-59ae1084fa14/crypt/debug\", \"/pool/ext/12d428b9-eedd-40c0-8928-1d92a73f181d/crypt/debug\", \"/pool/ext/08d4507c-e72e-4afd-9f03-5e28f4ad0909/crypt/debug\", \"/pool/ext/7ba4af02-bf8f-47c6-af79-d69fdad5d4bf/crypt/debug\", \"/pool/ext/62a187a2-0706-420e-a881-83ff781ad6c1/crypt/debug\", \"/pool/ext/5da68ecb-08f6-4895-925f-903c81000441/crypt/debug\", \"/pool/ext/3b36f3f1-29d0-4d11-8b6b-ae2a89620d43/crypt/debug\", \"/pool/ext/2780ea55-1799-4c3e-916a-9024b9586eaf/crypt/debug\", \"/pool/ext/bae019e4-4e95-466e-aa8a-f660660e26c1/crypt/debug\"] }","zone_name":"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543"}
{"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:35.531849525Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543\", bundle_id: 54a3329a-dfe0-4d98-904a-0afc9d2e7667 }, time_created: 2024-05-15T07:18:34.892497534Z, version: 0, cause: TerminatedInstance }"}
{"msg":"Halting and removing zone: oxz_propolis-server_b926d0d2-78b1-4d16-b1ed-d24aaeaf7543","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:35.531939217Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:780"}
{"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:37.284856115Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:37.312232365Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-15T07:18:34.891896712Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-15T07:18:37.312163043Z } }","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:37.38839296Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50316"}
{"msg":"unset_virtual_nic_host unimplmented","v":0,"name":"SledAgent","level":40,"time":"2024-05-15T07:18:37.390195827Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:620"}
{"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:37.390221788Z","hostname":"sock","pid":1555,"uri":"/v2p/6a12a6a1-9861-4fe8-bb03-f96ecb313186","method":"DELETE","req_id":"e909a894-9f95-4ab3-8e6b-b5686c5b6344","remote_addr":"[fd00:1122:3344:101::a]:50316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/283d897/dropshot/src/server.rs:878","latency_us":1698,"response_code":"204"}
{"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:37.650968672Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:491","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-05-15T07:18:34.891896712Z }, propolis_id: b926d0d2-78b1-4d16-b1ed-d24aaeaf7543, vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-05-15T07:18:37.312163043Z } }","instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49"}
{"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-05-15T07:18:37.654785782Z","hostname":"sock","pid":1555,"instance_id":"74b5d93e-2241-4319-8129-fe67108c8f49","component":"InstanceManager","file":"sled-agent/src/instance.rs:723"}