[ Jun 22 01:54:50 Enabled. ] [ Jun 22 01:54:50 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jun 22 01:54:50 Method "start" exited with status 0. ] [ Jun 22 01:54:50 Rereading configuration. ] [ Jun 22 01:54:50 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:50.584433581Z","hostname":"sock","pid":1562,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:50.584577073Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:50.637710568Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:50.637759019Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:50.750598707Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:50.759666914Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:50.767122357Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.36977753Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.507002687Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.507031298Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.507094369Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.50716343Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.50719848Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.507227011Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.510272247Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:51.510278987Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.510289557Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.510294047Z","hostname":"sock","pid":1562,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.510305207Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.510346658Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.510413899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.510429789Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.51044628Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.522862357Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.522904158Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.624527887Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.835535681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.835576902Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.835582402Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.835587392Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.835600042Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.835604262Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.835620692Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.844596858Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.844622488Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"ad11261d-76fd-447c-b4fa-06153786f4ac (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:51.933465193Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149258299Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.14931754Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149326731Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149341821Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149356351Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149361581Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149366721Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149373341Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149384851Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149390541Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149394741Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149398352Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149402761Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149415582Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149438022Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149442472Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149447192Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149451132Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149471983Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149500953Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149506153Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149527323Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149532414Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149537204Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149541484Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149547334Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149551304Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149555654Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149580304Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149584744Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149588294Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149593624Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149597564Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149616815Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149621965Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149626765Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149631245Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149635275Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149660025Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149664045Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149669376Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149673416Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149678176Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149685316Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149703616Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149707686Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149714936Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149718646Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149722617Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149726397Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149753417Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149758927Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149763107Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149766727Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149771257Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149778067Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149810288Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149815068Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149827018Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149831458Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149835678Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149840018Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149846108Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149850718Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149868749Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149874209Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149878669Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149895679Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.149909789Z","hostname":"sock","pid":1562,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.161552376Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:710"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.161586076Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999929189s","next_cleanup":"Instant { tv_sec: 1498, tv_nsec: 984437355 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.161658837Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3547"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.161664887Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.161719598Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.161732118Z","hostname":"sock","pid":1562,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165050909Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165199961Z","hostname":"sock","pid":1562,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165452904Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165459055Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165511476Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165558986Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165572516Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165581006Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165604967Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165615077Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165625767Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165644887Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165648647Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165652307Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165666368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.165704178Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166054704Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166221666Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166246716Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166289917Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166296757Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.16645955Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166581392Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166624172Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5afeac69-ea1f-40c0-9e56-34defe323774, rack_id: e8d1fa68-5f11-401f-90e7-316434459ea8, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166729874Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:46627","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 5afeac69-ea1f-40c0-9e56-34defe323774, rack_id: e8d1fa68-5f11-401f-90e7-316434459ea8, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166822235Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166831885Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166855126Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166859926Z","hostname":"sock","pid":1562,"file":"sled-agent/src/server.rs:44"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166863636Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166867116Z","hostname":"sock","pid":1562,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166877016Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166887296Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166897466Z","hostname":"sock","pid":1562,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166906997Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.166910897Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.169400944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3779"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.17970103Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.190190209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.199705063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.208947183Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.217724376Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.226309496Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.234894796Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.243483576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.252073956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.252122837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.252129537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.252134067Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.252141727Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.252149707Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.258448482Z","hostname":"sock","pid":1562,"sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.258489913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.258496233Z","hostname":"sock","pid":1562,"sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.258591205Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 54.336268ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.264580105Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2b723863-bab2-42cd-aa9c-41429d15a06e\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:54:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2b723863-bab2-42cd-aa9c-41429d15a06e\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_dfa37dd5-5aca-43c5-b94a-79f447e6cd21/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.272001537Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.276361814Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.293028206Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_dfa37dd5-5aca-43c5-b94a-79f447e6cd21/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.295040346Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_dfa37dd5-5aca-43c5-b94a-79f447e6cd21/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.295358801Z","hostname":"sock","pid":1562,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.30058404Z","hostname":"sock","pid":1562,"sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:381"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.3005919Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_dfa37dd5-5aca-43c5-b94a-79f447e6cd21/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.30059601Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.319558677Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 142.632083ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.320019495Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1adc6066-b279-4f7c-8607-7f74323707a8\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:54:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1adc6066-b279-4f7c-8607-7f74323707a8\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.374590291Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.46370738Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 90.538807ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.4643776Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bf7d1e44-fd92-4e69-92bf-7dfc4cc32209\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:54:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bf7d1e44-fd92-4e69-92bf-7dfc4cc32209\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.555447359Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 160.585281ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.561279007Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"659d50f9-0ab2-413e-97c5-b4e6dd98b68a\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:54:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"659d50f9-0ab2-413e-97c5-b4e6dd98b68a\" }"} {"msg":"Mounting oxi_dfa37dd5-5aca-43c5-b94a-79f447e6cd21/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.593482194Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.607994014Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.608100986Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.608132596Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.608164137Z","hostname":"sock","pid":1562,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.723130027Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 294.907588ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:52.723785927Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ede69149-2dd5-4f5d-b863-457629fe53a7\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:54:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ede69149-2dd5-4f5d-b863-457629fe53a7\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.020034011Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 156.020686ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.020933455Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"70056f44-75e7-41e2-9871-666cda28c509\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:54:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"70056f44-75e7-41e2-9871-666cda28c509\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.177989612Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 428.44053ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.17847867Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"52160060-ad8c-46e4-9ed0-37032e8e1c91\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:54:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"52160060-ad8c-46e4-9ed0-37032e8e1c91\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.303658325Z","hostname":"sock","pid":1562,"sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.304299395Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.304320585Z","hostname":"sock","pid":1562,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.304397466Z","hostname":"sock","pid":1562,"file":"oximeter/producer/src/lib.rs:282","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:282","interval":"30s","address":"[fd00:1122:3344:101::1]:48051","producer_id":"5afeac69-ea1f-40c0-9e56-34defe323774"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:53.305644735Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"166.208096ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.352037007Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.352071738Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.35218584Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:893","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.35219258Z","hostname":"sock","pid":1562,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.352940731Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.352979102Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.353007002Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.353017062Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.353234825Z","hostname":"sock","pid":1562,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.353407238Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.353429339Z","hostname":"sock","pid":1562,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.353442799Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3547"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.353457539Z","hostname":"sock","pid":1562,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:46627","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.524062762Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.524114392Z","hostname":"sock","pid":1562,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.524174023Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.524257974Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.524287365Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.524315155Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.524345086Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:53.525577874Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"732.737168ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:53.530238225Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.555911384Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45722"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.556068486Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.556150327Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"6ffa0d0f-f8fc-4263-a80d-4f060dbb49ef","remote_addr":"[fd00:1122:3344:101::1]:45722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.586672939Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:59060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.586773561Z","hostname":"sock","pid":1562,"uri":"/sled-role","method":"GET","req_id":"3ef488ba-176c-49c0-a962-58df5771b5d0","remote_addr":"[fd00:1122:3344:101::1]:59060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":27,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.588190702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.588220133Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.588570188Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.588766581Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:812"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.608001122Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 0ca0be89-7daf-4811-aba7-8c457d3a3c38, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool) }, OmicronPhysicalDiskConfig { id: 45cd4c95-7f1a-47ea-ab8b-1992ba01b256, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool) }, OmicronPhysicalDiskConfig { id: 9e81d759-c255-4e77-b3db-c42184daefae, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool) }, OmicronPhysicalDiskConfig { id: cd7328b2-8a48-438c-addb-ffb213b9a961, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool) }, OmicronPhysicalDiskConfig { id: 76a882d4-2193-425f-831b-03a6f2440ae5, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool) }, OmicronPhysicalDiskConfig { id: a2526934-ea03-4b44-8765-2d065214fdc1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool) }, OmicronPhysicalDiskConfig { id: 7b17484d-2f3c-4b13-b292-6d043ecf6309, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 37a51968-6b4b-4243-8e26-612da188294b (zpool) }, OmicronPhysicalDiskConfig { id: 707ebbb1-0ab2-4cda-a1f8-35f457120f88, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool) }, OmicronPhysicalDiskConfig { id: 7e35e199-e8ab-489a-a2f7-48621b997f08, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617330903Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617428205Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33423"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 0ca0be89-7daf-4811-aba7-8c457d3a3c38, pool_id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 45cd4c95-7f1a-47ea-ab8b-1992ba01b256, pool_id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 9e81d759-c255-4e77-b3db-c42184daefae, pool_id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: cd7328b2-8a48-438c-addb-ffb213b9a961, pool_id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 76a882d4-2193-425f-831b-03a6f2440ae5, pool_id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: a2526934-ea03-4b44-8765-2d065214fdc1, pool_id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 7b17484d-2f3c-4b13-b292-6d043ecf6309, pool_id: 37a51968-6b4b-4243-8e26-612da188294b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 707ebbb1-0ab2-4cda-a1f8-35f457120f88, pool_id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 7e35e199-e8ab-489a-a2f7-48621b997f08, pool_id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617645108Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617661128Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617666729Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617671439Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617681049Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617689399Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617694829Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.617699449Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.712652557Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:53.712685267Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:54.260417508Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"903.866948ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:54:54.500615065Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.683112137Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.847187301Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.847248102Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.847254932Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.847263062Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.847270193Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.847418755Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:54.847425005Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:55.213135321Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.697419009s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:55.426166946Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:55.64907496Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Automatically destroying dataset: oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:55.838847283Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:56.268996844Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 65.75401ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:56.447240293Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:54:56.652328327Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 90.113824ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:56.828556025Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 247.831991ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:57.209705635Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 223.867914ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:57.96306702Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:54:57.96576677Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:57.965845642Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.731817405s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:57.965859772Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.065591913s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:54:58.564776698Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_fbfdf456-46af-4ca5-8e50-42e804e95de3","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:58.828323158Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:54:59.031385161Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:59.037351411Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:59.037364152Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:59.037371042Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:59.037376102Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:59.037380242Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:59.277229333Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:54:59.277266983Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.426423659Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.668795299Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.668837849Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.668844689Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.66887785Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.66888655Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.669018042Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:00.669064203Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:55:01.150998518Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:01.151190461Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:02.11628154Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:55:02.678880837Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:02.678996169Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:02.67907057Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.595457719s"} {"msg":"Automatically destroying dataset: oxp_072ad6a6-48d3-47f5-b5ad-907cb6ccb208/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:02.679384675Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.407382275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2233"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.637776513Z","hostname":"sock","pid":1562,"component":"VmmReservoirManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:55:03.680365838Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_072ad6a6-48d3-47f5-b5ad-907cb6ccb208","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.748908686Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.756594262Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.756626222Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.756635523Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.756640352Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.756644283Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.765968014Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.765994104Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"21f0441c-e835-4662-89b4-8f9c50d54833 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.862458094Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.87006338Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.87009107Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.87009667Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.870132871Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.870149461Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.870319944Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.870326193Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.904630303Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_21f0441c-e835-4662-89b4-8f9c50d54833/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:03.912104196Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_21f0441c-e835-4662-89b4-8f9c50d54833","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.029380411Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.036953126Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.036994116Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.037003696Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.037022436Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.037031817Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.04716107Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.047191461Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.135509518Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.14228798Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.142326031Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.142334271Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.142354331Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.142362131Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.142508183Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.142514933Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.179400942Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_ce3883c7-6e59-4796-9a58-b1fefe1b0724/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.187074428Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_ce3883c7-6e59-4796-9a58-b1fefe1b0724","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.310612538Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.317468532Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.317502762Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.317509733Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.317517863Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.317522303Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.327168879Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.327193099Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"cadffa03-0f46-493d-807b-355e3bd965dd (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.419231733Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.42631061Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.426353141Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.42636177Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.426392571Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.426401611Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.426601584Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.426610294Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.462994085Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cadffa03-0f46-493d-807b-355e3bd965dd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.4699325Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_cadffa03-0f46-493d-807b-355e3bd965dd","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.592296922Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.599276898Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.599315379Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.599321259Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.599328399Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.599334409Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.608720371Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.608745142Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.70311241Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.711316304Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.711348925Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.711356145Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.711398976Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.711409836Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.711567378Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.711577758Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:04.71566725Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.747704175Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:55:04.751184678Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Automatically destroying dataset: oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.755891249Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_a5bbfc86-7c7e-4dd8-823e-791e9cd96508","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.878250231Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.886134451Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.886175062Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.886182242Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.886187402Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.886225352Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.896179273Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.896214154Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"37a51968-6b4b-4243-8e26-612da188294b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.990900086Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.99907539Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.9991018Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.99910732Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.999131031Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.999141001Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.999318994Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:04.999328484Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.035789576Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_37a51968-6b4b-4243-8e26-612da188294b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.043159017Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_37a51968-6b4b-4243-8e26-612da188294b","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.163443358Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.171241586Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.171297297Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.171305537Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.171311018Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.171315427Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.181049465Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.181088375Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.276128144Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.283832761Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.283872451Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.283886722Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.283914412Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.283924842Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.284094345Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.284101085Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.322508606Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_31de0ecb-89b0-4d10-a4da-a6cbbdd4101a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.331168258Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_31de0ecb-89b0-4d10-a4da-a6cbbdd4101a","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.447957095Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.454724978Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.454789029Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.454802829Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.454808089Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.454812559Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.464632618Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.464665028Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9f456447-b67b-4176-b7d7-23a75139f1da (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.561149959Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.56913838Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.56917589Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.569183551Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.569195241Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.569204531Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.569395764Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.569412384Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.603828965Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9f456447-b67b-4176-b7d7-23a75139f1da/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.610912872Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_9f456447-b67b-4176-b7d7-23a75139f1da","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.732108937Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.739117743Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.739158464Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.739195875Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.739365437Z","hostname":"sock","pid":1562,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: dfa37dd5-5aca-43c5-b94a-79f447e6cd21 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: ad11261d-76fd-447c-b4fa-06153786f4ac (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.739410738Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.739419058Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.73959012Z","hostname":"sock","pid":1562,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.739890745Z","hostname":"sock","pid":1562,"uri":"/omicron-physical-disks","method":"PUT","req_id":"84b2a2e2-4783-46f3-8928-ca5e4241c598","remote_addr":"[fd00:1122:3344:101::1]:33423","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12122406,"response_code":"200"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:55:05.753796176Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.769709896Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.769846888Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50619"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.770184593Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.770198924Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.822879221Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.822934152Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.873045031Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.873087911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.924272616Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.924308777Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.924345417Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.924392468Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.968496065Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.968537046Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:05.968541676Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:06.017598109Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:06.02097737Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:06.024822748Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:06.037872505Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:06.042410704Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:06.046250052Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:06.714032392Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:55:06.755002172Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:07.545205524Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:07.545259264Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:08.019566324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:08.494703857Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:08.494767458Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:08.494841139Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:08.49887075Z","hostname":"sock","pid":1562,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:08.498902641Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:08.503181836Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 60.41585ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:08.627561888Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 70.835362ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:08.633849984Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 66.939871ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:08.639942416Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 107.420011ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:08.878076561Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 75.792961ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:08.979314063Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 93.161928ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.022548177Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 178.04331ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.089029204Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 299.838639ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.11257975Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 216.248567ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.166125321Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 220.7691ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.303671873Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 558.043421ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.409364323Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 230.172662ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.453280758Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 1.053907929s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.53469588Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 576.439047ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.693545735Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 778.108543ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:09.977208279Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 1.191112659s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:10.278572341Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 1.132372837s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:10.600665547Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 1.423465327s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:10.768295435Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:11.360409168Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2250"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:11.360433898Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 1.45584085s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:11.478169391Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 1.478861983s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:11.740299339Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:feb4:f9af","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:11.908293072Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2257"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:11.914255442Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2318"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:12.033620989Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2503"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 764.792031ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:12.199925226Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:12.292896814Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2667"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:12.472232509Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2382"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:12.761087812Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2417"} {"msg":"wait for service Some(\"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c\") failed: Property not found. retry in 1.110618321s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:12.942862464Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8\") failed: Property not found. retry in 1.094806282s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:12.972660884Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:12.976777586Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2717"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:13.160935164Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2744"} {"msg":"wait for service Some(\"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431\") failed: Property not found. retry in 567.452117ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:13.228375215Z","hostname":"sock","pid":1562,"zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:13.384981126Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2611"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:13.55565766Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2411"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:13.566342822Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3803","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:13.566375692Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3819"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:13.566396492Z","hostname":"sock","pid":1562,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.094258783Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_70f6b296-e2a4-4dba-8ddf-037e755a5431"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.094356615Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_5874e87d-84fd-420e-8e71-3d603966202c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.094363895Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.094469486Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.094861482Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.095157347Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d4692e2b-5ddb-4950-a5c3-69ad6b403aa0","remote_addr":"[fd00:1122:3344:101::1]:50619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8325185,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.097012725Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.121583017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3827"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.129404155Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3883"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.294554085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3945"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.396049802Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.418273958Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3973"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.516071799Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:4000"} {"msg":"failed to write DNS configuration (will retry in 149.585055ms)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:14.622410378Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:569","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.640024435Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.671795246Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.703148521Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.703230272Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.70374612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.70376532Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781321884Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781372985Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781383525Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781391605Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781401535Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781409456Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781418566Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781441176Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781447036Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781452836Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781458756Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781465036Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781470786Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781479457Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781488007Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781497917Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781504587Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781510517Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781516827Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781526637Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781533137Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781539658Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.781545687Z","hostname":"sock","pid":1562,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.792467013Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.83853602Z","hostname":"sock","pid":1562,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.884146581Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.884158741Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.915961942Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.917240862Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.917246182Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.945395818Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.94551593Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52298"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.946576326Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.946583276Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.968429366Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.968470707Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.988919156Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:14.988964067Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.007183983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.007236683Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.007389016Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.007527168Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.019674442Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.05127662Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.05189987Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.05190603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.096548626Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 138, 38]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.096608816Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.097106224Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.097119474Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.097346887Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.098386513Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.121750367Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.140747915Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.757288248Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:15.75737967Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 58.352913ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:15.884345422Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 123.923259ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:15.964598296Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 260.326245ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:16.117603892Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 508.251654ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:16.387422167Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 427.34355ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:16.905604542Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 1.235025912s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:17.34282988Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:18.054739017Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47297"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:18.054876069Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"12e96ffd-f490-43fd-ae72-f8d0e4215aa0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47297","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 811.08479ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:18.587156117Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 1.040949042s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:19.407493735Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860\") failed: Property not found. retry in 691.499723ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:20.457751933Z","hostname":"sock","pid":1562,"zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.167294305Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.167377076Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.167713871Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.168009716Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"af1e7a89-38ac-49f7-ade4-e6ed92bf6fe0","remote_addr":"[fd00:1122:3344:101::1]:52298","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6222431,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.16965851Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.197799136Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.197938049Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41685"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.207022606Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.207173568Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"1a4fde8a-6b7b-40b6-9510-ae839bcdb42f","remote_addr":"[fd00:1122:3344:101::1]:41685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9078,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:21.209359471Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.350139223Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:21.350219124Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:43346"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.358754885Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.358865267Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"5b12afa7-5e5c-4d3a-97b5-82e7448cf076","remote_addr":"[fd00:1122:3344:101::1]:43346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008551,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:25.361826331Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.48985256Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.489988002Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35178"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.499254942Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.499442115Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"71922847-324c-4d52-991e-40e30bd0ddf1","remote_addr":"[fd00:1122:3344:101::1]:35178","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9264,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:25.502308938Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.662013126Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:25.662166218Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61862"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_89cdce4c-b444-4eba-b844-8541f2127860': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:29.670581928Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:29.670684499Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"19ea8391-080e-4e8c-a8ee-e6aa6b7c71fa","remote_addr":"[fd00:1122:3344:101::1]:61862","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008391,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:29.673895858Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:29.876751619Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:29.876896701Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61167"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:29.885659194Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"b5386ace-0d4e-444c-b0e5-d7b9be9b2805","remote_addr":"[fd00:1122:3344:101::1]:61167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8612,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:29.88739153Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:29.88739606Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.164696037Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.164824799Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:58444"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.173314678Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"955e0925-ab86-441f-9254-39dc663e0064","remote_addr":"[fd00:1122:3344:101::1]:58444","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8359,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.175076505Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:30.175083205Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.452483394Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.452595676Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.461144175Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"ecdf2b8a-d945-49cf-8351-ce46e9802224","remote_addr":"[fd00:1122:3344:101::1]:64231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8442,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.46280481Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:30.46280903Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.867145501Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.867264023Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:61621"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.875941995Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"9ea5c435-2658-4fdd-a37b-0f4dd3381f2d","remote_addr":"[fd00:1122:3344:101::1]:61621","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8560,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:30.877592589Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:30.87759842Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.003248219Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.003365511Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33931"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.025151221Z","hostname":"sock","pid":1562,"uri":"/timesync","method":"GET","req_id":"31fdb444-503c-4044-bfa5-5055ab5b1b5d","remote_addr":"[fd00:1122:3344:101::1]:33931","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":21637,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 1192464524, ip_addr: 71.19.144.140, stratum: 3, ref_time: 1719021331.3347027, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.026900318Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.026918588Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.055270557Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.05545023Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:62340"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.056377144Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.056383254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.076820643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.076855214Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.095558477Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.095606487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.113306815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.113336396Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.165921282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.165967313Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.21728162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.21731694Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.268526935Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.268562186Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.317868052Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.317901733Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.36857496Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.36861514Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.368866324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.368981866Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.44983229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.449872661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.449878961Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.449884681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.449889991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.542082117Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.546245929Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.550539545Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.554781929Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.559295307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.563758375Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.569174727Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.57399406Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.57795158Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:31.582643491Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:33.217189864Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"242.232368ms"} {"msg":"Profile for oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.095133219Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.095294101Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.103208671Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.103342543Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.163137648Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.163328881Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.232221714Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.232329205Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 51.293961ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:35.246959767Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 71.774654ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:35.254868977Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.258208697Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:35.258345899Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 145.222835ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:35.308090812Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 35.986998ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:35.321345213Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 70.206981ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:35.439284148Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 67.614191ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:35.749814119Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 202.895618ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:35.822675042Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 155.19767ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.087729334Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 223.729497ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.203894723Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 483.904333ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.257367693Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 592.52061ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.482188895Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 389.92021ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.637257263Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 59.196704ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.731933276Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 43.783363ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.837581325Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 460.037877ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.87555392Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 145.600414ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.888843261Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 128.093299ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:36.929363565Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 990.035046ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.060763914Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 129.407755ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.115149337Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 170.25018ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.199079198Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 765.610655ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.283065379Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 226.827773ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.3868607Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 267.243514ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.418877095Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 639.289717ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.455138974Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 1.186311438s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.638334247Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 751.664065ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:37.728063716Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 885.14226ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:38.060466298Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 791.805109ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:38.072636682Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 1.070548458s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:38.107851465Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 1.25555461s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:38.489284928Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 1.076164885s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:38.834627007Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 974.675536ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:38.873595377Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 1.122978691s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:38.954101935Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 1.053009935s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:39.18825122Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 792.52971ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:39.753442165Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 1.075498497s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:39.858452545Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 589.70099ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:39.919719163Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 1.287001037s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:40.087912699Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 1.058661258s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:40.252813475Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 745.337955ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:40.521940639Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 1.219359754s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:40.557105301Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 1.03788574s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:40.942173501Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 1.395491967s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:41.275356315Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 871.770562ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:41.319501203Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 1.304294791s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:41.382836882Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 1.270985816s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:41.784798326Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 1.105692025s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:41.98851612Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 1.372794613s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:42.200142134Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 1.03434407s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:42.681705884Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679\") failed: Property not found. retry in 1.14497049s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:42.696746271Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 601.806763ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:43.067040317Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 1.272810901s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:43.10353475Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7\") failed: Property not found. retry in 1.020434012s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:43.582708743Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f\") failed: Property not found. retry in 697.629655ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:43.679904255Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70\") failed: Property not found. retry in 1.086370494s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:43.725792769Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 848.283789ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:44.3857623Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 933.491354ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:45.2438801Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 980.857493ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:46.187973942Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 1.077774939s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:47.58504158Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:48.582631932Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:48.582837345Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"b178dde3-5e06-4a30-8d39-94c8b9822c3b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51242","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 615.593454ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:48.679306285Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:49.068170542Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"516.170751ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 1.254335426s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:49.30648523Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 973.39185ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:50.576434214Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 1.274947354s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:51.559938372Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d\") failed: Property not found. retry in 989.266534ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:52.843510763Z","hostname":"sock","pid":1562,"zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.856330455Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_d61772a4-765b-4b4e-afed-61558f738679"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.856402866Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_e0033f3a-9c2b-45dc-a8fa-b7874eccf34d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.856426247Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_6add796f-d0b5-4da7-88bb-bab76d9366c7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.856430827Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_257fb649-c62a-4980-966e-32cb72b90d70"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.856434767Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_9bfa240c-63a3-4943-b5f7-96ec8098339f"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.856444357Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.857433112Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.858285345Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a69c6e2a-4c8c-449f-ac3c-4c2d7629e46b","remote_addr":"[fd00:1122:3344:101::1]:62340","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":22228762,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.88967888Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:51476"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:53.91808118Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3367"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:54.138299944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3385"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.455743373Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:3408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.455837234Z","hostname":"sock","pid":1562,"uri":"/cockroachdb","method":"POST","req_id":"2f7e273b-3afa-47eb-a457-36de4b61927d","remote_addr":"[fd00:1122:3344:101::1]:51476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3566118,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.486835473Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.487014626Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55634"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.487504963Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.487512104Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.51101317Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.51106862Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.532025128Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.532088838Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.550759911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.550813262Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.570225076Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.570270927Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.590928469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.59097332Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.608185021Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.608227001Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.62400229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.624050161Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.64184472Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.641891171Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.707828879Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.70787983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.776299886Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.776352106Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.837924928Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.837971819Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.891030442Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.891080723Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.958352782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:57.958405312Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.032780688Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.032840999Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.107104874Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.107167124Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.17952938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.179596031Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.242173768Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.242237029Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.303272293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.303325954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.357966661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.358016862Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.414955964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.415025865Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.415207148Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.41534877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512092284Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512133154Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512139745Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512144305Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512148585Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512151945Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512156095Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512159505Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512162755Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512167025Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512172375Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512176475Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512181725Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512185795Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512190905Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512195015Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512208555Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512213776Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512217926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.512330417Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"19a01518-a56b-424e-b391-1c1b455cacbf","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized)]","latency_us":1025169,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:58.512606532Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"75.317472ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"19a01518-a56b-424e-b391-1c1b455cacbf\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:55:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"19a01518-a56b-424e-b391-1c1b455cacbf\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.588749604Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.589222561Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.589230521Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.608480823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.608531484Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.624555836Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.624618147Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.640437297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.640487768Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.656234976Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.656280837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.672180867Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.672220458Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.688960951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.689004802Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.707230808Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.707288539Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.723741718Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.723784398Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.741334774Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.741390605Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.763034823Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.763082444Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.782605319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.78264647Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.802424919Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.80246617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.823851613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.823896544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.843762455Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.843802846Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.863402822Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.863448783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.881833811Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.881895182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.899912995Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.899952825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.917832576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.917876647Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.941357412Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.941419863Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.963070301Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.963118622Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.963319275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:58.963489037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057035434Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057083524Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057089954Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057093424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057098704Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057102084Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057106395Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057110775Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057114985Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057118175Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057122275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057126575Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057130925Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057134975Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057139055Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057144155Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057158225Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057162565Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.057166685Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.057296498Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5808c184-d384-4bde-8071-41a590fca6e4","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized)]","latency_us":468302,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.05747058Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"62.3504ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5808c184-d384-4bde-8071-41a590fca6e4\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:55:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5808c184-d384-4bde-8071-41a590fca6e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.120664817Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.121003632Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.121014592Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.140570428Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.140607009Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.157043717Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.157090668Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.17373745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.173777941Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.19024757Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.190281651Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.207108375Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.207143376Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.22329486Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.223331181Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.240614423Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.240657773Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.258059197Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.258102867Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.274565516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.274604117Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.292415917Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.292453297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.309435575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.309470005Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.328162128Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.328194819Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.34544973Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.34548533Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.363218699Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.363254839Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.382829255Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.382862156Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.403213294Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.403248025Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.422384404Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.422421395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.440100813Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.440131343Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.460829276Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.460869216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.479469728Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.479516769Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.479754712Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.479892594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571824876Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571916017Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571927648Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571938128Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571947708Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571953478Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571960588Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571965978Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571972768Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571978198Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571983678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571988958Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.571995719Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.572002369Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.572011059Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.572018589Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.572031459Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.572035609Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.572040909Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.572128431Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"a0b344b6-d508-4410-8247-fd24e19a5251","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized)]","latency_us":451319,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:55:59.572352234Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"182.92773ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0b344b6-d508-4410-8247-fd24e19a5251\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:55:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a0b344b6-d508-4410-8247-fd24e19a5251\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.756602353Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.756958559Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.756966089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.780099659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.78015054Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.802715591Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.802765182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.821619578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.821667918Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.841534319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.8415803Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.86008957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.860133511Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.879535614Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.879584135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.899181732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.899237903Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.917791073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.917836974Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.937729495Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.937776696Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.9565352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.956575731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.976051886Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.976097826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.997284977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:55:59.997335118Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.018583589Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.01863393Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.038114445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.038166746Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.058758407Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.058803698Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.077049815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.077094155Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.095058367Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.095102348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.112870697Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.112918527Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.132796338Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.132845549Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.151641544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.151696954Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.151883817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.152005469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250448089Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.2504929Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.25049873Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.25050382Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.25050801Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.25051221Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.25051643Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250521561Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250525941Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250531021Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250534971Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250539121Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250543361Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250547531Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250551611Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250555731Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250572161Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250575651Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250579782Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.250659223Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1d14202f-b301-4618-8f88-43fb8cf58756","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized)]","latency_us":493880,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.250861606Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"87.864826ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1d14202f-b301-4618-8f88-43fb8cf58756\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:55:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1d14202f-b301-4618-8f88-43fb8cf58756\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.339229653Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.340127647Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.340134517Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.361056694Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.361091415Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.380730902Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.380767352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.399926132Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.399961163Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.419341066Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.419372857Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.439777266Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.439811426Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.46058594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.46061829Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.481379005Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.481420036Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.501145894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.501179455Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.522308484Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.522346365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.542982048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.543017758Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.562727286Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.562762947Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.582614988Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.582651798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.602063712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.602095942Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.621400995Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.621436395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.641885505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.641916575Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.661861877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.661899618Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.680877675Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.680905976Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.700615344Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.700645344Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.720725108Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.720763719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.740958085Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.740995445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.741209838Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.74134443Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845207763Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845285604Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845294584Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845300264Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845305964Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845311674Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845315904Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845321405Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845326585Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845333635Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845339155Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845345825Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845351105Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845356845Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845362255Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845369065Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845374565Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845385575Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.845390966Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:00.845487207Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b7fa4f2c-03b2-4423-9f1b-6c2de0f919c3","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized)]","latency_us":506132,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:00.84570457Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"290.387384ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b7fa4f2c-03b2-4423-9f1b-6c2de0f919c3\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b7fa4f2c-03b2-4423-9f1b-6c2de0f919c3\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.137278394Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.137584279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.137592209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.162190801Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.162238322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.184245055Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.184290056Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.203260433Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.203306524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.222088728Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.222144329Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.242528267Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.242578188Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.26250742Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.262549661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.280970449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.28103035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.300257681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.300299652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.31929652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.31934847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.339230711Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.339278322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.359559889Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.35960592Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.379254448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.379303948Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.400620341Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.400661741Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.42103767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.421085831Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.441439799Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.44148513Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.460757981Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.460797361Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.480032282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.480078913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.498766876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.498812537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.519796024Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.519849755Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.541392451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.541450952Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.541641325Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.541765107Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.643598079Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.643653679Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64365981Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64366506Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64367005Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64367345Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64367664Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64368147Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64368475Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64368791Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.6436921Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64369632Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64370032Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64370455Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64370865Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64371275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.64371774Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.643735551Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.643739911Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.643842842Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d2767d06-a04a-4e81-863f-7d35b5f0c8e5","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized)]","latency_us":506447,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:01.644063506Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"165.607748ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2767d06-a04a-4e81-863f-7d35b5f0c8e5\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2767d06-a04a-4e81-863f-7d35b5f0c8e5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.810272562Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.810637547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.810648538Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.833721937Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.833761037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.855165892Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.855212032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.876514935Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.876563005Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.901485463Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.901526103Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.923474346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.923512696Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.944203979Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.94423949Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.964268053Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.964317784Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.984158584Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:01.984197035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.004522913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.004573553Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.025460319Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.02549981Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.045103927Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.045138067Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.065635988Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.065690868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.087117873Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.087156433Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.107351479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.10738954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.126667562Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.126705882Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.147366245Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.147403965Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.167074643Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.167112844Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.186589099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.186624649Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.206248076Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.206286187Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.225960935Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.226003695Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.226245569Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.226390811Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331504143Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331578664Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331604924Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331610844Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331618084Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331623744Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331630894Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331636914Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331642515Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331649555Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331654405Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331662435Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331668195Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331675275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331683385Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331692906Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331703975Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331718616Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.331725786Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.331862688Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"e1c4fb30-18b9-47ae-8280-f87354d2cb96","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized)]","latency_us":521430,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:02.332104982Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"154.400609ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e1c4fb30-18b9-47ae-8280-f87354d2cb96\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e1c4fb30-18b9-47ae-8280-f87354d2cb96\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.487343591Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.488367137Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.488380017Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.511802312Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.511852142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.531087724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.531135174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.549718666Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.549776007Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.569067799Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.569120129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.591424187Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.591475378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.613701814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.613749665Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.63518434Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.6352467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.655750171Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.655816422Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.677905756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.677956747Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.700507198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.700566339Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.723239402Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.723288293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.745654782Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.745714973Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.766835213Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.766883973Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.789234281Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.789282412Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.812842039Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.81289594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.83468954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.834741461Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.855759479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.85581146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.877225413Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.877272534Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.900704799Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.90075671Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.924383938Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.924447968Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.924681472Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:02.924860895Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033233605Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033293526Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033314416Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033322417Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033329527Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033334817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033345587Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033351867Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033359067Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033364757Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033371657Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033376968Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033382217Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033389238Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033394758Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033402008Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033407508Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033423118Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033427258Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.03351413Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"4a72f187-3b0a-4ba2-8649-80a3765dcf20","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized)]","latency_us":546040,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.033749893Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"210.937889ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4a72f187-3b0a-4ba2-8649-80a3765dcf20\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4a72f187-3b0a-4ba2-8649-80a3765dcf20\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.246010416Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.246373882Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.246382162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.270181132Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.270231793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.292937547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.292988748Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.313705001Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.313759502Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.333783205Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.333827166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.3539294Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.353981351Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.374750596Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.374799896Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.394350822Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.394406063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.415828107Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.415874528Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.435770439Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.43581417Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.456499972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.456553683Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.477058574Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.477108074Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.497776907Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.497837868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.520867167Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.520911528Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.540037877Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.540082388Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.563679135Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.563725566Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.583905711Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.583948702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.606086487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.606136918Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.626678649Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.62674395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.647037037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.647081238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.668917478Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.668972619Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.669174812Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.669293174Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766339423Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766396354Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766422144Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766429184Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766434944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766442225Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766448125Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766453625Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766460895Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766467015Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766472495Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766478175Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766482215Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766488025Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766495445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766501045Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766506615Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766520156Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766533926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:03.766691288Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"bc3ae7d2-0d29-4590-856c-2fd5b8dcd043","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized)]","latency_us":520532,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:03.766914512Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"398.600974ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bc3ae7d2-0d29-4590-856c-2fd5b8dcd043\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bc3ae7d2-0d29-4590-856c-2fd5b8dcd043\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.166259557Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.166628323Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.166654983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.190417733Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.190468704Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.213169087Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.213224038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.234561651Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.234612952Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.254806708Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.254856608Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.274991253Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.275040524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.296125493Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.296167284Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.315608448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.315654739Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.33552557Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.33557288Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.355748086Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.355797496Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.375430484Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.375485505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.395259494Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.395306475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.415730544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.415787155Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.435521814Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.435565294Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.45511945Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.45516615Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.476841028Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.476889869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.499122296Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.499167016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.519698747Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.519740418Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.540669615Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.540711965Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.560973592Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.561024143Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.581837648Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.581902919Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.582133672Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.582279804Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.640516216Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"796.574525ms"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682424851Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682485252Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682493602Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682500802Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682506412Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682512082Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682517652Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682524572Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682529922Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682535492Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682540932Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682546422Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682553313Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682562353Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682572813Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682603003Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682609273Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682616023Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.682621574Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:04.682774216Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"f444fc1b-7153-49e7-b452-eca0704d16a5","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized)]","latency_us":516362,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:04.68306662Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"526.252704ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f444fc1b-7153-49e7-b452-eca0704d16a5\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f444fc1b-7153-49e7-b452-eca0704d16a5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.210596006Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.210939352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.210949322Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.233995571Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.234055451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.253454205Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.253517296Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.273402827Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.273454648Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.292904482Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.292954093Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.311629516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.311684047Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.331160892Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.331208962Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.351191375Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.351237316Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.369989049Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.37003259Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.390058193Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.390100004Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.409280224Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.409325305Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.428717409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.428767069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.449254409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.44930876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.4724458Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.472507731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.493010311Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.493057432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.516613568Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.516660489Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.537495404Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.537545745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.557404376Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.557454757Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.578544906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.578595237Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.598954705Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.598998886Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.619993773Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.620051034Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.620245147Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.620380129Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.723998598Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724040919Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724045449Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724050759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724055009Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724058309Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724063259Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724067479Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724072439Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724077589Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724081759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724086719Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724090879Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724095109Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.72410004Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.72410456Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.72412063Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.72412591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.72412995Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:05.724225841Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d2c2a4d1-ade3-4718-b95d-f07e9a03e763","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized)]","latency_us":513491,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:05.724429585Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"562.9726ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d2c2a4d1-ade3-4718-b95d-f07e9a03e763\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d2c2a4d1-ade3-4718-b95d-f07e9a03e763\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.289134373Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.289514249Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.289527869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.312463656Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.312501857Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.332575731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.332613972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.353283274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.353317375Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.374503246Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.374539066Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.394418537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.394455498Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.415818381Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.415852362Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.436107658Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.436158439Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.457784776Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.457816856Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.479564976Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.479596076Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.500802067Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.500843308Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.521046724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.521079844Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.539686096Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.539717406Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.56174868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.56179268Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.581638861Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.581670031Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.603234108Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.603266378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.622943256Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.622990917Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.642770576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.642805377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.664205761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.664238931Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.683815378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.683846828Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.704767795Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.704808355Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.705050209Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.705183781Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803608861Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803666622Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803674592Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803681402Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803686772Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803692322Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803699263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803704763Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803711863Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803717733Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803725043Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803730993Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803738193Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803743933Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803750683Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803754793Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803780134Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803787684Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.803794074Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:06.803927016Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d096950f-a06c-4f90-8e1b-43f5d30037fa","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized)]","latency_us":514613,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:06.80417402Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"717.023161ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d096950f-a06c-4f90-8e1b-43f5d30037fa\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d096950f-a06c-4f90-8e1b-43f5d30037fa\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.522540714Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.52294723Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.5229574Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.546540567Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.546593958Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.568273626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.568335257Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.588404561Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.588452062Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.609139755Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.609184906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.62798871Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.628037621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.648580642Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.648628073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.668093357Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.668143558Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.688351454Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.688426955Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.709421113Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.709470544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.730466712Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.730518182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.753067354Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.753123485Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.774021851Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.774065972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.796864357Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.796915228Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.821233556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.821271906Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.843189198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.843223589Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.863271722Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.863312453Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.883670821Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.883707701Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.904861742Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.904897152Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.927036777Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.927072518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.948484702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.948535503Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.948782757Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:07.948917099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.050921913Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051012424Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051021265Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051028654Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051033975Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051041335Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051052375Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051059145Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051066425Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051072295Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051077975Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051082085Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051087535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051094366Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051099876Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051105336Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051119356Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051123606Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051130796Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.051258218Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9a120f15-891f-415d-899b-5314a2946862","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized)]","latency_us":528536,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:08.051477591Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"594.556576ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9a120f15-891f-415d-899b-5314a2946862\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9a120f15-891f-415d-899b-5314a2946862\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.647082037Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.647407752Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.647414792Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.671421006Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.671463697Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.692750999Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.69279954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.712711011Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.712749211Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.732679263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.732718314Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.752549054Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.752584714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.771795275Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.771827506Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.793029847Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.793080718Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.813861422Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.813894113Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.834962022Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.834997562Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.856106412Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.856147212Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.877506516Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.877544006Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.899673181Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.899710112Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.920286153Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.920322344Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.941819709Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.94185754Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.961474487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.961517008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.982362593Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:08.982402164Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.0032758Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.0033145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.023562217Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.023601047Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.042614805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.042652676Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.063637994Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.063685774Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.063886497Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.064020159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166252417Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166303568Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166309528Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166314988Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166318318Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166323658Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166327678Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166332728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166337018Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166342068Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166346098Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166350138Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166355078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166359158Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166364129Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166368159Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166383729Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166387119Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166390359Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:09.16646371Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9d5ab79d-540e-4a96-830e-ad7ab5f49060","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized)]","latency_us":519246,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:09.166671863Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.1932626s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9d5ab79d-540e-4a96-830e-ad7ab5f49060\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9d5ab79d-540e-4a96-830e-ad7ab5f49060\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.360349683Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.360723579Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.360731019Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.383329791Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.383369282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.403175072Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.403208492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.422646116Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.422678927Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.441908248Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.441951779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.460123153Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.460155974Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.478930878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.478966928Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.4975677Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.497605741Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.515455651Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.515490972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.534226475Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.534257576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.553438226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.553469736Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.572104329Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.572132099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.592189173Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.592219963Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.61048917Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.61052304Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.629875853Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.629903764Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.650204661Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.650232601Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.668828453Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.668857383Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.686847366Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.686876266Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.705449237Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.705479718Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.724354923Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.724390254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.743135398Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.743172268Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.743372391Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.743500523Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840718605Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840810096Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840819037Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840824677Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840831457Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840837337Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840844637Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840850317Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840857267Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840862857Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840870217Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840876097Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840882858Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840889558Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840896758Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840902348Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840911308Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840918408Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.840925628Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:10.84105535Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"9cd6049e-f862-449f-a276-ba6f3e7572ac","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized)]","latency_us":480545,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:10.841308764Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.067761913s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9cd6049e-f862-449f-a276-ba6f3e7572ac\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9cd6049e-f862-449f-a276-ba6f3e7572ac\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.910173134Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.910501689Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.91051209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.934082666Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.934138237Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.95547717Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.955525731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.976217114Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.976269855Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.994622293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:11.994670424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.015031122Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.015084563Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.034595628Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.034643639Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.054209075Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.054268626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.072770186Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.072812667Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.092283171Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.092323432Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.110872733Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.110921544Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.13051991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.130572721Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.151076421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.151125732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.170941122Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.170985663Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.189151468Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.189199098Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.208501461Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.208543271Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.22824764Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.22829654Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.246149071Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.246191291Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.265067597Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.265119588Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.285412485Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.285453786Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.305788394Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.305847384Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.306071508Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.30620201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.40004069Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400094921Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400101781Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400107361Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400111551Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400115591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400120591Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400124642Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400129552Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400133582Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400138572Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400142652Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400147922Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400152022Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400157172Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400162532Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400165712Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400179292Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400182812Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:12.400268604Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1600599c-405a-43a3-a240-0d478e2463d5","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized)]","latency_us":489951,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:12.400474837Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.22732225s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1600599c-405a-43a3-a240-0d478e2463d5\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1600599c-405a-43a3-a240-0d478e2463d5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.628396002Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.629333367Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.629350707Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.650119971Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.650161452Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.668825694Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.668858865Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.688165487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.688199058Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.707352968Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.707383878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.72598176Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.72601869Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.743706868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.743739849Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.762014765Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.762064456Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.780128599Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.7801615Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.799242899Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.79927982Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.818291757Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.818323268Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.8363089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.83634225Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.85546804Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.855512861Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.873417932Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.873455122Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.893522826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.893552867Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.911923085Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.911955365Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.930481876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.930512976Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.949217879Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.94924805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.968349599Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.968385029Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.986422813Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:14.986462603Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:15.004131571Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:15.004168771Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:15.004374094Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:15.004498506Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096309536Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096382037Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096390267Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096397137Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096402838Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096410058Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096414298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096419808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096425358Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096430498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096435988Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096442778Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096446668Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096452298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096456298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096463118Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096468488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096480029Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096486649Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:15.096618571Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"b139ef72-0ed3-4583-98ef-9429ac3b9981","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized)]","latency_us":468038,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:15.096836454Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.120105071s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b139ef72-0ed3-4583-98ef-9429ac3b9981\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b139ef72-0ed3-4583-98ef-9429ac3b9981\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.217780763Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.218565735Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.218573525Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.243873618Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.243918639Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.264374408Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.264426209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.285026191Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.285075312Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.303811216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.303852766Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.323884159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.32392686Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.342615493Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.342654644Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.361222395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.361270825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.379798146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.379837667Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.39856448Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.398607201Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.417431036Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.417489937Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.436026377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.436069658Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.454812331Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.454869282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.473440693Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.473483893Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.492309098Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.492345859Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.510507294Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.510546665Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.529097115Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.529143156Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.548448438Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.548485359Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.567532617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.567573588Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.589720983Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.589764524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.610798152Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.610851153Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.611048716Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.611164488Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706276268Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706317728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706332178Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706337529Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706342809Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706347078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706352119Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706356199Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706361099Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706364389Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706369209Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706374149Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706378159Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706382989Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706387809Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706390959Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706394349Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706408319Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.70641283Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:16.70647588Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1d422a05-1fc0-4465-bce9-ff09fcb119ed","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized)]","latency_us":488548,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:16.706690484Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.355756782s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1d422a05-1fc0-4465-bce9-ff09fcb119ed\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1d422a05-1fc0-4465-bce9-ff09fcb119ed\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:18.615686502Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47351"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:18.615780304Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"17a6673d-a3a1-4a53-aa63-d3c68e34f9e9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47351","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.063831477Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.064172982Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.064181412Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.087352933Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.087388403Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.107176973Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.107211283Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.125168705Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.125204246Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.143017095Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.143051146Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.162071904Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.162102324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.179894214Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.179926304Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.20015285Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.200179501Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.218803653Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.218830553Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.236897226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.236926687Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.256632335Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.256666656Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.275848056Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.275909767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.29658469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.296631591Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.315539437Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.315588298Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.333777573Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.333823324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.354086221Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.354130131Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.373327342Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.373376753Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.391880993Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.391929633Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.409324257Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.409371797Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.426649669Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.42669243Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.445606226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.445658297Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.44585772Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.445986892Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537283863Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537324324Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537329834Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537333344Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537337414Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537341644Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537346874Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537354565Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537359705Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537363714Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537368125Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537372905Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537376895Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537380845Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537393765Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537398155Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537403205Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537407295Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537412205Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:19.537494956Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"59aeff7c-ac7b-47ad-8a45-1382440c39d1","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized)]","latency_us":473524,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:19.537693599Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.41964439s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"59aeff7c-ac7b-47ad-8a45-1382440c39d1\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"59aeff7c-ac7b-47ad-8a45-1382440c39d1\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:20.489485318Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.142189717s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:20.958731571Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:20.959030916Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:20.959038346Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:20.984447381Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:20.984493441Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.004441153Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.004477014Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.025024075Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.025053815Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.044739473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.044773824Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.062644424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.062678545Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.081683913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.081713233Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.101930279Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.10196443Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.120219206Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.120252767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.138937719Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.13897282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.157366908Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.157408539Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.177426892Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.177469253Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.196031824Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.196070254Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.215531449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.215572959Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.235786116Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.235827966Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.255181099Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.2552357Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.274126166Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.274162126Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.29219501Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.29223231Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.310778161Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.310823341Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.329319421Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.329357512Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.347748391Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.347789271Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.348024475Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.348198937Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441672912Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441735413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441744014Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441749834Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441755684Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441761304Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441769394Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441774974Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441780484Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441788834Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441796914Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441801704Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441818115Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441829445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441835005Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441840535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441851425Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441855635Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.441859715Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:21.441907006Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1e518bf3-6d3f-41c0-a439-bfe2ccfb23f8","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized)]","latency_us":483038,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:21.44215271Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.905904133s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1e518bf3-6d3f-41c0-a439-bfe2ccfb23f8\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1e518bf3-6d3f-41c0-a439-bfe2ccfb23f8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.348490858Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.348934185Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.348943885Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.372530202Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.372605783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.392740208Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.392795029Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.412884273Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.412942084Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.43119433Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.431255591Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.449197942Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.449248732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.467772773Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.470069528Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.485794996Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.485857727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.506856335Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.506910605Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.527403626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.527452896Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.546370663Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.546427934Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.565779447Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.565835567Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.585970082Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.586027233Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.605351186Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.605409816Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.624422674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.624478115Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.644547079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.64460626Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.663840621Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.663893152Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.682338951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.682389842Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.700284903Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.700330473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.718343086Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.718401417Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.737156761Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.737239062Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.737410455Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.737542507Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831416938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831498419Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831508439Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831523599Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.83153196Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.83154072Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.83154897Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.83155445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.83156132Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.83156609Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.83157949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831587311Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831604511Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831610001Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831617461Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831629341Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831634511Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831639501Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831647772Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:23.831652752Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d8cc7f84-b33f-4e6f-bb9d-c56343f58839","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized)]","latency_us":482966,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:23.831841764Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.131356068s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d8cc7f84-b33f-4e6f-bb9d-c56343f58839\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d8cc7f84-b33f-4e6f-bb9d-c56343f58839\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:28.964621835Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:28.96501293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:28.965023361Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:28.988043089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:28.98811504Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.009325301Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.011690727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.030152727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.030227678Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.050550445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.050601686Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.07066695Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.070721251Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.091306842Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.091375043Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.113279175Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.113334036Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.132832981Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.132887632Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.152281155Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.152332766Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.173471446Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.173543047Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.193794954Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.193858035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.212208172Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.212267533Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.232626242Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.232677833Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.251900613Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.251959274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.271769824Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.274202461Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.291851008Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.291902679Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.310775895Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.310832586Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.330746087Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.330799798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.349165216Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.349225957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.367215779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.36728828Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.367480133Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.367613005Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463340534Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463435795Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463444986Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463451976Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463458196Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463464446Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463472476Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463480096Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463485856Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463493166Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463503307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463510647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463546457Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463561597Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463567327Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463576748Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463582648Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463588138Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463593768Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:29.463650559Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"38d90fb0-dc18-4174-b19b-65cdbeb7f160","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized)]","latency_us":498840,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:29.463869142Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.387355816s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38d90fb0-dc18-4174-b19b-65cdbeb7f160\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"38d90fb0-dc18-4174-b19b-65cdbeb7f160\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.852340787Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.853291871Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.853301951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.877389276Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.877458387Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.897589272Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.897640753Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.917720547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.917780727Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.936825116Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.936875377Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.957439738Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.957491419Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.977207547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.977278038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.996250286Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:32.996302786Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.017092801Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.017151142Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.038496845Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.038560276Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.059887069Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.05994192Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.078655623Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.078708964Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.098792618Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.098848409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.120825101Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.120879082Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.140609691Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.140661302Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.162239278Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.16231368Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.182355963Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.182413124Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.201872608Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.201922979Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.222609372Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.222666003Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.243231105Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.243283825Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.263524492Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.263615393Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.263779785Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.263908078Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359505125Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359584826Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359596836Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359606836Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359615776Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359623756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359630027Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359637127Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359642927Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359658147Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359667487Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359674027Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359678277Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359693028Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359700458Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359706188Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359710288Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359714448Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359724988Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:33.359747988Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"63a05298-f77b-405d-be95-1dd46be2084b","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized)]","latency_us":507247,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:33.359960521Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.358422511s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63a05298-f77b-405d-be95-1dd46be2084b\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63a05298-f77b-405d-be95-1dd46be2084b\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:37.227656206Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.760877121s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.279069302Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.279384197Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.279394407Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.300990324Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.303532443Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.322956547Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.325361013Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.344944819Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.34696179Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.366437055Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.366490686Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.386107993Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.388432758Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.405685769Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.407686999Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.42161837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.421668771Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.437834176Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.437898947Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.456667731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.456719462Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.477136771Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.477199092Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.498543555Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.498601856Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.517835297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.517893868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.535820659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.53587494Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.557295454Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.557351105Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.577411559Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.57747062Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.59799358Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.598051021Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.616626852Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.616691333Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.636429262Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.636480133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.656323293Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.656373924Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.675827379Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.67589158Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.676058302Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.676191724Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773379065Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773453997Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773463587Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773471687Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773477307Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773485147Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773491077Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773496587Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773503817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773510938Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773516947Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773536728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773544508Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773556558Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773573638Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773578808Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773583188Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773590579Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773595119Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:38.773637099Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"bb9793b1-0ad8-4e81-8428-01d797461058","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized)]","latency_us":494411,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:38.773826812Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.579423782s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb9793b1-0ad8-4e81-8428-01d797461058\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bb9793b1-0ad8-4e81-8428-01d797461058\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.354606778Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.354965743Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.354981863Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.378036972Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.378092733Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.39901253Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.401404356Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.417820045Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.417883876Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.436219393Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.436269124Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.456293467Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.456344348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.474905199Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.47495998Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.493996798Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.494057019Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.512500968Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.512555469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.531641208Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.531702459Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.549661101Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.549726732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.569261887Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.569315148Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.588072752Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.588123693Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.609133451Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.611596978Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.629161714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.629228105Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.647456911Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.647511652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.666940266Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.666990837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.685290334Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.685340795Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.703299776Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.703358657Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.722805522Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.722859083Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.741652977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.741715178Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.741959092Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.742091384Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835351616Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835435907Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835445797Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835454237Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835468357Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835477088Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835483948Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835505078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835521728Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835534858Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835543498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835555359Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835563959Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835568859Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835575459Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835580109Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835584919Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835589519Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835597419Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:42.835602359Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"24c852ae-97f0-47d8-a42f-7b8ab681b657","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized)]","latency_us":480844,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:42.835811993Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.557943111s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"24c852ae-97f0-47d8-a42f-7b8ab681b657\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"24c852ae-97f0-47d8-a42f-7b8ab681b657\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.205796923Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51349"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.205903965Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"ef293424-3541-43ea-a76d-a8d00bbcbfe1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51349","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":56,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.394622812Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.394999537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.395036838Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.417272465Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.419834153Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.437901687Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.437956848Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.457214399Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.45726424Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.476212817Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.476267698Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.494832479Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.494889589Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.513110355Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.515449991Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.531715767Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.531784378Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.550283278Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.550332939Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.567702282Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.567757913Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.586806461Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.586860652Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.605739638Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.605793079Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.624161077Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.624208977Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.642088518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.642138369Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.660844352Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.660899653Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.680292396Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.682717583Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.700052945Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.700114006Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.71819395Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.718244681Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.738473957Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.738529698Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.757587646Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.757643837Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.776162667Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.776224628Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.776392081Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.776503503Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869260347Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869339658Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869349648Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869356738Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869365898Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869371269Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869378259Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869383339Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869406399Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869415089Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869425319Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.86943711Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.86944408Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.86945073Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.86945594Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.86946053Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.86946707Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.86947196Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.8694768Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:48.86948194Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"d4a2890a-9d70-4e27-ba7b-66e8426b4bed","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized)]","latency_us":474686,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:48.869685753Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"7.212534165s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d4a2890a-9d70-4e27-ba7b-66e8426b4bed\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d4a2890a-9d70-4e27-ba7b-66e8426b4bed\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:55.024432213Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"7.599836216s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.083989204Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.08434819Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.084360219Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.109002595Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.109064358Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.130048348Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.130112896Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.151107287Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.151170045Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.17196204Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.172021038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.193311615Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.19337036Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.212034231Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.212101742Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.234730208Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.234791418Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.255049826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.255124518Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.278001459Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.278076085Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.300988061Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.301051474Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.321895573Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.321948543Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.341818093Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.341951951Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.364330274Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.364398151Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.387907447Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.387964298Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.412657345Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.41272554Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.436003887Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.436069035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.461347199Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.461409206Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.485910507Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.48905754Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.506690012Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.506744978Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.529324897Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.529456385Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.529646672Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.52976958Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637601213Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637749328Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637755103Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637761169Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637765135Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637769456Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637787026Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637792416Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637796607Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637800498Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637805823Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637811251Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637816257Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637820457Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637824199Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637833686Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637838955Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637844083Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.637852905Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:56:56.637856767Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"60280cb9-a905-4bd6-9ac3-e226e66aca41","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized)]","latency_us":552176,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:56:56.638048376Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.056964807s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"60280cb9-a905-4bd6-9ac3-e226e66aca41\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:56:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"60280cb9-a905-4bd6-9ac3-e226e66aca41\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.726201909Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.72670864Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.726720875Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.750920868Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.750979997Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.772861692Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.772922171Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.792021438Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.792070394Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.810968305Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.813219263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.82951483Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.829567921Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.847600317Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.847645121Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.86673264Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.866784756Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.885551267Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.885599249Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.904481763Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.904542326Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.923726967Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.923784745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.942510557Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.942566227Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.961005779Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.961060098Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.979939773Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:07.98237257Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.001686664Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.001738527Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.02191527Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.021967143Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.041337731Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.041404473Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.059768617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.05981359Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.078938702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.078993031Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.096892178Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.096946179Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.116825758Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.116900524Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.117195841Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.117338746Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211330647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211426292Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211438386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211448586Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211459414Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211465208Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211495472Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211501603Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211505465Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211509065Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211514297Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211524731Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211530347Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211534443Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211538072Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211542029Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211551422Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211555126Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211559401Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:08.211563385Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1e819109-029b-4df4-9a93-9f7c93a95211","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized)]","latency_us":483861,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:08.211780514Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"8.366329578s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1e819109-029b-4df4-9a93-9f7c93a95211\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:57:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1e819109-029b-4df4-9a93-9f7c93a95211\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.602267404Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.602846922Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.602857262Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.623183209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.623240163Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.643558633Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.643609578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.660511017Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.660554105Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.678192606Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.678246261Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.694187046Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.694241159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.712976205Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.713032878Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.72978907Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.729828737Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.748240086Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.74828985Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.766746594Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.766799892Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.783881809Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.783924034Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.800801263Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.800851917Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.821802655Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.82184805Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.839061826Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.839117252Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.857424258Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.857468556Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.875085016Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.875132407Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.892680471Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.892728958Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.912660513Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.912718264Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.930666698Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.930718956Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.948382765Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.948424391Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.967490162Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.967547613Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.967817681Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:16.967943599Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059757431Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059908255Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059922581Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.0599328Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059942625Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059952206Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059973356Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059980866Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059988263Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.059993701Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060013782Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060030441Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060036216Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060043877Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060051077Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060063246Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060069067Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060074739Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060082333Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:17.060087771Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"1223ffce-711b-4419-a1f3-eddd583855c9","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized)]","latency_us":456343,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.060306221Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"10.952916816s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1223ffce-711b-4419-a1f3-eddd583855c9\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:57:16 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1223ffce-711b-4419-a1f3-eddd583855c9\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:17.720082648Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"12.874590984s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:18.296105345Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42933"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:18.296230206Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"92250725-6810-44e3-9129-1a698d155db9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:42933","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":33,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.044128206Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.045177145Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.045187607Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.066484783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.066639036Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.086414038Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.086462133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.106222073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.106270466Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.125214133Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.125269005Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.144078226Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.146150316Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.164222476Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.164271705Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.182648035Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.182701988Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.200579064Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.200624496Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.218911487Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.219033713Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.239159936Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.23921929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.25762511Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.257672783Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.275992209Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.276038063Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.295398107Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.295446793Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.315174996Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.315293114Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.333694082Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.333738557Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.352282578Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.352410437Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.371236537Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.371283459Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.389352071Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.389407929Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.409630406Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.4096818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.42926037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.429310141Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.429664413Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.429818625Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521182386Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521270766Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521282448Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521303354Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521310291Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521319019Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521325236Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521333008Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521339308Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.52134392Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521350511Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.52135518Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521367939Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521379208Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521385612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521391499Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521401287Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.52140799Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521414374Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:28.521452907Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"3f7467ba-9c95-47e7-a6af-b690f5c94c75","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized)]","latency_us":475813,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:28.521699519Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"28.345740106s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f7467ba-9c95-47e7-a6af-b690f5c94c75\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:57:28 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f7467ba-9c95-47e7-a6af-b690f5c94c75\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:45.704075418Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"16.150457027s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:48.41116945Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47867"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:48.411267326Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"38ba6559-e5fa-4501-8186-5930a9c5c24d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47867","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:56.945096222Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:56.945374702Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:56.94539048Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:56.967698934Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:56.967759901Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:56.989454091Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:56.989506179Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.012012785Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.01209073Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.03125273Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.031307397Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.053038659Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.053091769Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.071246812Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.071304722Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.0906648Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.090730022Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.109492674Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.109542821Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.127974818Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.128028641Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.147714498Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.147769595Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.167881505Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.167942997Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.186988341Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.187043252Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.205647469Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.205699229Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.225033414Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.225090198Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.243917553Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.24397732Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.26254617Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.262600687Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.280751314Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.280804386Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.299094098Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.299150714Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.317115196Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.317159737Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.336714208Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.336777246Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.337099826Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.337276828Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430714174Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430786221Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430795662Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430810663Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430818979Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430826863Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430847563Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430854585Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430951707Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430967166Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430972857Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.430997373Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.431004048Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.431008623Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.431015167Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.431021495Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.431028001Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.431033926Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.431038436Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:57:57.431052331Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"5be8e479-2414-4e32-a3ba-17961a1073f1","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\", TimeNotSynchronized), (\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\", TimeNotSynchronized), (\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\", TimeNotSynchronized), (\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\", TimeNotSynchronized), (\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\", TimeNotSynchronized), (\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\", TimeNotSynchronized), (\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\", TimeNotSynchronized), (\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\", TimeNotSynchronized), (\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\", TimeNotSynchronized), (\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\", TimeNotSynchronized), (\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\", TimeNotSynchronized), (\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\", TimeNotSynchronized), (\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\", TimeNotSynchronized), (\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\", TimeNotSynchronized), (\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\", TimeNotSynchronized), (\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\", TimeNotSynchronized), (\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\", TimeNotSynchronized), (\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\", TimeNotSynchronized), (\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\", TimeNotSynchronized)]","latency_us":484498,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:57:57.43126268Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"25.364080322s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5be8e479-2414-4e32-a3ba-17961a1073f1\", \"content-length\": \"124\", \"date\": \"Sat, 22 Jun 2024 01:57:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5be8e479-2414-4e32-a3ba-17961a1073f1\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:16.97452666Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"43.081724603s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:18.526551694Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:18.526699306Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"2bc4073b-7dde-407b-9a98-e3540d0fb954","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:44384","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5874e87d-84fd-420e-8e71-3d603966202c, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 89cdce4c-b444-4eba-b844-8541f2127860, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 5e2bd555-b928-44d1-8037-f679b623da86, kind: Service { id: 89cdce4c-b444-4eba-b844-8541f2127860 }, name: Name(\"ntp-89cdce4c-b444-4eba-b844-8541f2127860\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 138, 38])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: d61772a4-765b-4b4e-afed-61558f738679, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 257fb649-c62a-4980-966e-32cb72b90d70, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: d722cbf1-822b-421d-89f9-027b7d805ad2, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: b7f02198-e503-4542-8190-5c4bb0e37c4d, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1006b3bf-a9da-42f1-a646-5acd47e2044d, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: c1140ebf-da6d-488c-9487-59995dca8c1d, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: b6d73b34-7b4b-422b-bbba-34d59ea472c1, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: d1b133f5-fbc7-44a8-8ec2-655dac17b863, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1a734084-94e5-4a00-9ec0-114b8f518752, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External } } } }, OmicronZoneConfig { id: d2326324-7a68-4938-b232-00e6ade1d475, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External } } } }, OmicronZoneConfig { id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External } } } }, OmicronZoneConfig { id: 42f65a84-5d90-4925-babc-94e82788d5dd, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External } } } }, OmicronZoneConfig { id: d7191489-4945-412a-8e62-cdb44b265fb3, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External } } } }, OmicronZoneConfig { id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External } } } }, OmicronZoneConfig { id: bfb9dd76-a681-40e9-8b40-b51030886e8a, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.866406348Z","hostname":"sock","pid":1562,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.867369037Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.86737813Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c0dbdf8-8a9c-4830-b857-f2e7841fd2c8, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.891712402Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.891787703Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 5874e87d-84fd-420e-8e71-3d603966202c, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.916520409Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.916593385Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 70f6b296-e2a4-4dba-8ddf-037e755a5431, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.940079059Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.940135918Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d61772a4-765b-4b4e-afed-61558f738679, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.96317089Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.965384159Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9bfa240c-63a3-4943-b5f7-96ec8098339f, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.982075449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:22.982133725Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e0033f3a-9c2b-45dc-a8fa-b7874eccf34d, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.003970519Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.00402541Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6add796f-d0b5-4da7-88bb-bab76d9366c7, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.023544495Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.023595449Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 257fb649-c62a-4980-966e-32cb72b90d70, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.043509637Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.043564191Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 244b3e14-8cd3-4efe-a576-185d13c0ab68, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.06264715Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.062708182Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d722cbf1-822b-421d-89f9-027b7d805ad2, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.081892671Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.081946832Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c1140ebf-da6d-488c-9487-59995dca8c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.100503384Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.100561191Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 95c930b9-a06e-4c29-85e2-f9adf9ceff58, dataset_name: DatasetName { pool_name: ZpoolName { id: fbfdf456-46af-4ca5-8e50-42e804e95de3 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.119961434Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.120014844Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d2326324-7a68-4938-b232-00e6ade1d475, dataset_name: DatasetName { pool_name: ZpoolName { id: 072ad6a6-48d3-47f5-b5ad-907cb6ccb208 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.13937795Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.139432297Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0763b6fe-0de5-42ee-9c35-1035fae9b1f4, dataset_name: DatasetName { pool_name: ZpoolName { id: 21f0441c-e835-4662-89b4-8f9c50d54833 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.157939932Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.158000645Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: ffd9b6d3-de54-4abe-989f-4928a4837a3c, dataset_name: DatasetName { pool_name: ZpoolName { id: ce3883c7-6e59-4796-9a58-b1fefe1b0724 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.177779277Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.177827081Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 42f65a84-5d90-4925-babc-94e82788d5dd, dataset_name: DatasetName { pool_name: ZpoolName { id: cadffa03-0f46-493d-807b-355e3bd965dd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.196135033Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.196193458Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d7191489-4945-412a-8e62-cdb44b265fb3, dataset_name: DatasetName { pool_name: ZpoolName { id: a5bbfc86-7c7e-4dd8-823e-791e9cd96508 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.215121962Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.215175775Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2779e4bd-8e06-4f16-81f2-871e04c735ad, dataset_name: DatasetName { pool_name: ZpoolName { id: 37a51968-6b4b-4243-8e26-612da188294b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.233755298Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.233812524Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 128ab32a-f7c9-407c-aa78-c1f58f1fe9e7, dataset_name: DatasetName { pool_name: ZpoolName { id: 31de0ecb-89b0-4d10-a4da-a6cbbdd4101a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.251456276Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.251508214Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bfb9dd76-a681-40e9-8b40-b51030886e8a, dataset_name: DatasetName { pool_name: ZpoolName { id: 9f456447-b67b-4176-b7d7-23a75139f1da (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.269810595Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.269867484Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.2701484Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.270282784Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.388649453Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.388988608Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.389020268Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.38905445Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.399875451Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.413373612Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.413654894Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.428228119Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.428437664Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.428463745Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.42850238Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.612454652Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.612732052Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.612781722Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.612818416Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.612860032Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.612905586Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.624913986Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.624947576Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.625065459Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.6303206Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.634839632Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.639793944Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.644868791Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.649708165Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.654559172Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.793460226Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.798227302Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.802759791Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.808024884Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.813057149Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.817990329Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.822880379Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.828071467Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.832465635Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.837041646Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.842356997Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.847804667Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.854039243Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.859764943Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.865563835Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.871827605Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.87704751Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.882478949Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.888407921Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.895393076Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.905018092Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:23.972281414Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.024070817Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.148707733Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.208521722Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.326807563Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.33015752Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.330187558Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.434417234Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 184, 113]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.434508529Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.437735803Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.437764323Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.552726554Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 135, 8]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.55281152Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.555346275Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.555364331Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.658724576Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 199, 166]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.658800655Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.663669093Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.663704269Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.845568883Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 210, 93]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.845754726Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.85383759Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:24.853857437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.377952736Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 148, 170]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.378263889Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.385277278Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.38529572Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.386032034Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.386050296Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.386337038Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.386350773Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.417518099Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.544245111Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.545146842Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.645994347Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.651197568Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.651216768Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.651266953Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.651275119Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.651433025Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:26.651445915Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:27.61771169Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:27.638931354Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:27.665050802Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:27.665079471Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:27.676250488Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:27.706635064Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:28.24536422Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.386564075Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.447230693Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.506961999Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.667538048Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Configuring new Omicron zone: oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.776549171Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.816182124Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.837425692Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Installing Omicron zone: oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.850968733Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:34.856831987Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.618389515Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.620042556Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.620078894Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.628080359Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.861654596Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.861865151Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.873155756Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.873630132Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.923287027Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.924612537Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.924846869Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.932052981Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.940574445Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.941209526Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.949606979Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:35.950062302Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:36.308542065Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:36.308685438Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 25.140641ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.333879908Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 55.527869ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.346633042Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 74.957432ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.360385203Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 36.805462ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.374697961Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 100.240133ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.387971798Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 44.985324ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.403441088Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 99.860248ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.415306557Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 143.184498ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.426957818Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 44.444518ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.436148034Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 65.981262ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.446035182Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 143.111594ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.456584945Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 52.230413ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.4679801Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 137.685558ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.481092422Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 59.734077ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.492659222Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 25.116007ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.505267706Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 209.9095ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.521078614Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 149.376158ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.568930389Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 293.069779ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.657729652Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 82.262337ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.668360965Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 220.36514ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.677283206Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 291.443592ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.684928647Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 257.069703ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.697758961Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 156.682341ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.709643577Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 55.086736ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.719263503Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 48.961693ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.728529159Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 143.759485ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.743817052Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 350.092311ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.758029791Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 234.013782ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.771768263Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 244.959219ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.787566094Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 70.524368ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.799058244Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 282.461505ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.8806974Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 253.788303ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.892594425Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 397.567833ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.9065813Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 524.195315ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.917795646Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 279.011178ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.965901815Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 354.31031ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.978381715Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 378.58634ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:36.98918268Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 558.347079ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.019965365Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 388.077308ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.047541104Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.19475466s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.121172719Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 592.981676ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.158177988Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 766.613808ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.175538966Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 645.941351ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.257867224Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 421.31751ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.318089204Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 1.008269368s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.347888513Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.009633561s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.386140485Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.009112182s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.45078152Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 927.890712ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.462460056Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 870.796551ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.592025447Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 858.624425ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.753911246Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 654.546163ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.769078783Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 653.272353ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.920448215Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 759.895138ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:37.960246095Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.426284543s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.333203287Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 1.071110042s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.36974489Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 562.483ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.403231117Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 803.381134ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.415834648Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 863.281743ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.436787975Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.461613629s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.47612967Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 501.515346ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.487402891Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 862.15156ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.585948383Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 542.36275ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.625499042Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 1.499316879s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.73374445Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 560.640373ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.977132565Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 1.441240772s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:38.999775797Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 1.044124486s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.18026503Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 616.31982ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.230696593Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 962.762858ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.311828281Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 806.401316ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.474215579Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 580.294356ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.519662317Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:39.5680809Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 612.023915ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.578235093Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:39.580469267Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:39.606025174Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:39.625222815Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:39.782229381Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.103115173s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.798103852Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:39.798351093Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.034315318s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.861265748Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 33.603836ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.904798407Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 29.024288ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.933171906Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 920.353517ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.950986216Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 83.256705ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.963862981Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 140.975518ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:39.974488832Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 221.037186ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.060374372Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.202658428s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.114687544Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 50.365062ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.12625498Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 295.003533ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.137237796Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 149.177291ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.188431691Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 653.791812ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.229427801Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 1.033426422s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.242776023Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 803.997699ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.254990668Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.303249639s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.28887083Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 673.204775ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.302448226Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 221.363485ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.317442263Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 162.431911ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.349288125Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 343.643057ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.446618088Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 986.358631ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.457752867Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 201.46809ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.526834352Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 574.008739ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.551231872Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 513.087324ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.742811872Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 1.000788079s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.802339733Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.104840298s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.883871367Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 515.473489ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.893581664Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.343038831s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.932173508Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 896.71478ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.944939552Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:40.978568581Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 1.062823312s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:40.990661174Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:40.990927818Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 1.373775355s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.071894514Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 606.936449ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.137831092Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 38.51199ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.226940021Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 987.841791ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.269262866Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 131.67722ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.278463771Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 656.74767ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.288727036Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.057228799s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.332251195Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 1.336786793s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.4211456Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 199.709903ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.432033725Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 706.285488ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.472443669Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.232656823s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.605986368Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 364.579097ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.647374155Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 684.513634ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.766089362Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 1.283895662s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.82081594Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.165504022s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.85643757Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 747.877217ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:41.961382238Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 742.335148ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.002984948Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 863.226516ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.026064347Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 864.28945ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.069111419Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 817.251612ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.193677385Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 1.145252734s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.270766002Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:42.271413141Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:42.271706606Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.15363688s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.294943117Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.146685376s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.403412029Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 665.63281ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.458939119Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 884.253664ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.471956227Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 71.113732ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.54091394Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 78.157787ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.691282547Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 1.018179687s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.741581368Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:42.78202212Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 615.599385ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.793400742Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 1.339587728s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.814542753Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 196.609487ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.832018267Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:42.832116999Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 548.013542ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.85269745Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 1.177667165s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.904270651Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 1.433529228s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:42.946035143Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 1.032947017s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.025327783Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.242977482s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.039351986Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 245.407126ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.049214995Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 25.557644ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.057698953Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:43.05812449Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:43.058241302Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 60.503448ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.094100125Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 613.68534ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.114864759Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 519.661924ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.137748585Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 133.229233ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.164603972Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 680.050645ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.347771304Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 62.852178ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.396142256Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 266.127714ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.453592666Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 945.329976ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.471000799Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.312555614s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.491315137Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 713.669946ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.506370505Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 1.215382498s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.555560979Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 770.722371ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.591708837Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 99.218057ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.614738965Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 717.318172ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.634143929Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 672.909643ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.695187633Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 175.231362ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.770729067Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 831.271666ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.789544732Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 1.41358161s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.81056157Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 1.142326603s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.82603936Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 582.42882ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:43.962590251Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 1.373494611s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.045425995Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 948.36944ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.071128614Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 1.324383261s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.095064536Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 934.842634ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.187932111Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.313375546s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.234497507Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 510.732733ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.296519075Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 880.835604ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.364122849Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.228239917s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.377247448Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 700.833669ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.388067121Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 891.066021ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.399595556Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 875.91203ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.427716952Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 410.481143ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.558082975Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 515.027729ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.634119706Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 502.14119ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.78566431Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 630.222184ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.817726585Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 509.388505ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.82992923Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 1.04337332s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.978131064Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 1.058236273s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:44.985681918Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 1.468943936s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.028036129Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 1.285621292s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.097515351Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 521.295884ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.132404349Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 730.764388ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.156644256Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 1.31784331s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.232104328Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.157037863s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.253508082Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 1.132041591s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.296260839Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 770.239365ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.303276455Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 1.230928133s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.312420824Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.437382459s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.346654262Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 1.46118389s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.426322818Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 726.394923ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.434885738Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 952.054034ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.456131186Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 880.858519ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.558187514Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.159117055s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.616524397Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 933.732944ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.662594054Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 1.081021397s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:45.89682228Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 967.662949ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.030778708Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 1.28984985s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.053853747Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 874.790935ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.083600628Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 1.14357624s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.169981485Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 1.219647143s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.393315295Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 716.710813ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.417722395Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 547.414293ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.426056471Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 1.265898121s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.437377693Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 962.425669ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.448189046Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 1.185642354s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.506820014Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 1.119023431s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.565482192Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 784.753104ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.578772423Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 1.36096597s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.609062181Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.428115781s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.790490547Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.304522063s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.804417139Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 1.022940612s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.901795753Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 550.873428ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.975870314Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 591.586839ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:46.990942972Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 664.002024ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.011624926Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 1.053650773s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.028112655Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:47.030124535Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:47.030175436Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:47.030312978Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:47.041312635Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.327809649s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.152617825Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 1.16509345s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.332635294Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 664.274897ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.366590888Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 1.127965088s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.399278183Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 27.41446ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.422768069Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.143918796s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.447024371Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 47.936365ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.470050844Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 71.14494ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.543411115Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 995.864697ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.626571984Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 114.834153ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.641654642Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.276363499s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.658907463Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 687.818896ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.680203375Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 106.753928ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.696807527Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 1.197227676s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.719414559Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 590.988543ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.744378487Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 1.243670296s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.768079676Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 1.410120636s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.804623719Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 274.134534ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.837826842Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 280.692543ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.901387814Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 537.952048ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.947868218Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 1.417942623s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:47.988937999Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 759.875449ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.047638078Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 584.068092ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.099817148Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 950.71501ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.127520937Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 273.603231ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.145507039Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 889.23571ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.200959659Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 583.579881ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.241221098Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 710.968194ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.356745936Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 917.935528ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.383648744Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 1.163100536s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.431530479Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.334138277s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.491239483Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 822.523573ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.502433462Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 1.432875602s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.513850285Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 911.393705ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.540022861Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.12331354s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.601771066Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 1.306622421s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.631191501Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:48.651276155Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:48.65687855Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"0c3f92bd-38f4-44c2-b50b-c8f9a9381b7f","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48310","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3989,"response_code":"200"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 737.419314ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.6945523Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 900.004665ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.817980319Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.127007738s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.836498739Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 744.95808ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.928148737Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.144166372s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:48.944554915Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 1.10591497s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.022418633Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 1.443828058s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.079755132Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 526.803456ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.090152269Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 803.781386ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.099941127Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 915.735416ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.227036781Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 513.694095ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.311993126Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 892.045419ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.336099391Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 1.037387673s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.419072337Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 1.391453814s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.443826372Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 1.155158741s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.461216085Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 1.23612511s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.608788159Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.380762047s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.629235009Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 782.623723ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.685763785Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 1.006404541s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.727949403Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.253072718s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.739297355Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.452453267s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.83871434Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 1.281507302s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.849834848Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 925.638209ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.917616394Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 1.049599837s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.951810842Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 1.412289298s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.963946596Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.465134811s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:49.975709664Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.358565356s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.101714701Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 542.060263ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.141783338Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 643.809879ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.155654607Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 956.17916ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.239981914Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 558.050296ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.470679756Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 1.290411562s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.481669592Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 1.113005261s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.533896073Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 944.699488ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.629073844Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 1.326011347s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.696006367Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 502.339379ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.745920573Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 932.566481ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.814104425Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 1.218059978s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.846644087Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 931.565438ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.861975379Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 604.354258ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:50.8798597Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.01424226s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.00598503Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 1.347396229s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.017515544Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.02723951s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.027528145Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 1.499221994s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.039236383Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 1.177001485s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.14802016Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 855.328983ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.209201676Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 1.309041336s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.26166409Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 1.171048188s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.303617974Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 998.741659ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.388928496Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863\") failed: Property not found. retry in 1.305487668s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.459702677Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1\") failed: Property not found. retry in 1.21158778s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.477203372Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 508.861358ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.49821749Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d\") failed: Property not found. retry in 620.377247ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.593297829Z","hostname":"sock","pid":1562,"zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475\") failed: Property not found. retry in 1.269091016s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.664293634Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7\") failed: Property not found. retry in 1.44396597s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.762307238Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a\") failed: Property not found. retry in 1.298495255s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.793096164Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 1.079565563s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:51.809495222Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 1.013910658s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.027433821Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3\") failed: Property not found. retry in 909.087159ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.04124296Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58\") failed: Property not found. retry in 1.255022615s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.05375024Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 559.64084ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.069771632Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 972.752534ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.084604007Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 577.719381ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.10001783Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c\") failed: Property not found. retry in 680.639678ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.341410864Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752\") failed: Property not found. retry in 521.283478ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.385535352Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d\") failed: Property not found. retry in 1.033430285s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.415621047Z","hostname":"sock","pid":1562,"zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 879.082889ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.491689429Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad\") failed: Property not found. retry in 1.458572296s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.562679093Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d\") failed: Property not found. retry in 1.272007882s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.592435494Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4\") failed: Property not found. retry in 1.180704873s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.666742759Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68\") failed: Property not found. retry in 1.045600797s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.703714128Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 785.191851ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:52.945389509Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 1.132889189s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:53.275282941Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2\") failed: Property not found. retry in 998.808942ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:53.376857058Z","hostname":"sock","pid":1562,"zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd\") failed: Property not found. retry in 577.960828ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:53.408554147Z","hostname":"sock","pid":1562,"zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 1.444765765s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:53.758086379Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 717.064274ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:54.446958056Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\") failed: Property not found. retry in 1.452305906s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:55.182633743Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 758.473249ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:55.219563222Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f\") failed: Property not found. retry in 594.252384ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:55.994695655Z","hostname":"sock","pid":1562,"zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663302516Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_bfb9dd76-a681-40e9-8b40-b51030886e8a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663355107Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_2779e4bd-8e06-4f16-81f2-871e04c735ad"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663361567Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_0763b6fe-0de5-42ee-9c35-1035fae9b1f4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663367597Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_b7f02198-e503-4542-8190-5c4bb0e37c4d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663372287Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_external_dns_d722cbf1-822b-421d-89f9-027b7d805ad2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663377647Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_oximeter_1006b3bf-a9da-42f1-a646-5acd47e2044d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663381827Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_external_dns_244b3e14-8cd3-4efe-a576-185d13c0ab68"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663386427Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_clickhouse_c1140ebf-da6d-488c-9487-59995dca8c1d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663408437Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_b6d73b34-7b4b-422b-bbba-34d59ea472c1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663417988Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_1a734084-94e5-4a00-9ec0-114b8f518752"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663425078Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_59cebf87-1ddd-4cf3-8dfe-af47c91c765f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663433298Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_d2326324-7a68-4938-b232-00e6ade1d475"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663437738Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_ffd9b6d3-de54-4abe-989f-4928a4837a3c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663455208Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_42f65a84-5d90-4925-babc-94e82788d5dd"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663460818Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_d7191489-4945-412a-8e62-cdb44b265fb3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663469689Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_95c930b9-a06e-4c29-85e2-f9adf9ceff58"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663474389Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_88d03fa6-91e4-44cd-bd2f-3ca495b76e2c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663478598Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_128ab32a-f7c9-407c-aa78-c1f58f1fe9e7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.663483349Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_d1b133f5-fbc7-44a8-8ec2-655dac17b863"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.666576376Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.668610346Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.674878261Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"PUT","req_id":"76a8443e-3fea-4b86-94bb-8c2fa50ddfed","remote_addr":"[fd00:1122:3344:101::1]:55634","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":33765911,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.699157539Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.699174259Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.699357582Z","hostname":"sock","pid":1562,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.703757718Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.710070554Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:56.766950535Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:57.442777605Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:57.529690451Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:57.599928114Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:57.779225888Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:57.885583288Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:58.174996339Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:58.493296047Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:58:58.744939706Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:58.759443096Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:58.96385979Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:59.123242543Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:59.21153047Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:59.556127145Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:58:59.649292926Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:58:59.748248254Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:58:59.94290546Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:59:00.156126688Z","hostname":"sock","pid":1562,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"115.033929581s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:59:00.751386977Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:00.784193415Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:01.571620494Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:59:01.645895039Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:59:01.755564269Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:59:02.759850001Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:02.896556481Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T01:59:03.645734141Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/5afeac69-ea1f-40c0-9e56-34defe323774): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-22T01:59:03.763853799Z","hostname":"sock","pid":1562,"component":"ProbeManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:03.810311033Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/e8d1fa68-5f11-401f-90e7-316434459ea8/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:04.844775572Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:07.672757824Z","hostname":"sock","pid":1562,"component":"NexusNotifierTask","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.108276198Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45591"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.109849872Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.200990462Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"6d1d6482-8724-4e03-86ce-ec81bd04f12b","remote_addr":"[fd00:1122:3344:101::a]:45591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":92497,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.204003358Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.204086269Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.204736669Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.205115065Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"d85045e9-df55-4a29-9b89-1413c3287356","remote_addr":"[fd00:1122:3344:101::a]:45591","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1372,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.260223879Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58165"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.263315326Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.264203109Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.26425527Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.26427682Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.264295911Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.264318941Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.264337751Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.264366872Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c56bbdbf-b541-4fe3-ae4e-93101e9a6959","remote_addr":"[fd00:1122:3344:101::c]:58165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":4088,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288292453Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51878"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288435435Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288473426Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288509087Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288519997Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288538317Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288557297Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288572097Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.288588298Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"341acae9-4a08-40ba-9ed8-be8520b07b81","remote_addr":"[fd00:1122:3344:101::c]:51878","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":249,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.290554367Z","hostname":"sock","pid":1562,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.58532175Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33644"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.585366501Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.673404294Z","hostname":"sock","pid":1562,"uri":"/inventory","method":"GET","req_id":"333fd201-b15e-47be-9395-d752b6255752","remote_addr":"[fd00:1122:3344:101::c]:33644","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":88002,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.673510536Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33476"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674314628Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674399379Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674567772Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674652683Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674718954Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674725694Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674769315Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674799045Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Reading ledger from /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674807145Z","hostname":"sock","pid":1562,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674833586Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.674856676Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ad7fa258-aa3e-4e06-8421-c9d4be7f9a02","remote_addr":"[fd00:1122:3344:101::a]:33476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1301,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:08.67512824Z","hostname":"sock","pid":1562,"uri":"/omicron-zones","method":"GET","req_id":"b90187d1-f665-4e9b-a872-98ec9e96c9dc","remote_addr":"[fd00:1122:3344:101::c]:33644","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1452,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.157655726Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47199"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.157933671Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.157991971Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.158079313Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.158088063Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.158106213Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.158123443Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.158146134Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:10.158173354Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"b9dd4aee-f8a6-4f93-a565-ef8dd61c90e3","remote_addr":"[fd00:1122:3344:101::b]:47199","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":458,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:18.701804575Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33150"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:18.702242512Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"9228786b-ed51-4178-b078-bda4f4f81fa5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33150","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.515595002Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.516292303Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"df839789-c548-474b-ae3d-d96e554f8f3c","remote_addr":"[fd00:1122:3344:101::a]:58036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":676,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.759018447Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.85037475Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c900d991-adb7-4c5f-903e-416fef419cbc","remote_addr":"[fd00:1122:3344:101::a]:48381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91287,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.851865363Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3e1190c2-49b1-4b58-a024-22f098974114","remote_addr":"[fd00:1122:3344:101::a]:48381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1269,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.888607799Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.888908893Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.890030321Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.890994035Z","hostname":"sock","pid":1562,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:34.891028596Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"PUT","req_id":"102e0ddb-c9dc-4982-98c9-86a4d1712aca","remote_addr":"[fd00:1122:3344:101::a]:48381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9965,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.220149566Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50439"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.220377369Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"fcdf92b9-4e36-4706-806a-4542dfc0ea34","remote_addr":"[fd00:1122:3344:101::c]:50439","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":209,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.261327599Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47022"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.308675675Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"aaee6a66-7852-40c1-a09e-de6c3d81ec43","remote_addr":"[fd00:1122:3344:101::c]:47022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47288,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.308798917Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40529"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.30896078Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"1804a9a3-2fef-40ec-bdda-7efa20a5120e","remote_addr":"[fd00:1122:3344:101::b]:40529","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":135,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.309758341Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8df828f8-ce35-4e9b-82cc-945c968eacd3","remote_addr":"[fd00:1122:3344:101::c]:47022","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":742,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.314507703Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62601"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.361758049Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2c8352eb-88d3-4368-bb8d-f031b79bd482","remote_addr":"[fd00:1122:3344:101::b]:62601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47167,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:36.362249076Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"63ab91e9-197e-44d1-b96d-06f539e1fcc3","remote_addr":"[fd00:1122:3344:101::b]:62601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":217,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:48.745315238Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55301"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T01:59:48.746310353Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"e3f3cede-f0f2-4894-9f2e-9ff91d140b9e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55301","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":953,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:04.986238969Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52983"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:04.986561044Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b55f117d-ad2f-46b3-bab4-aba556b92970","remote_addr":"[fd00:1122:3344:101::a]:52983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":312,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:05.064112928Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64062"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:05.110569571Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"2e51de0d-9e3a-4d99-b311-462a888c042d","remote_addr":"[fd00:1122:3344:101::a]:64062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46412,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:05.111551196Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"18bee1b1-20f5-4102-b822-9d8a280e8d39","remote_addr":"[fd00:1122:3344:101::a]:64062","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":795,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.056154185Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.085913496Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35184"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.086099988Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"751cba98-113a-46c5-8b38-8b6b8a375b07","remote_addr":"[fd00:1122:3344:101::b]:35184","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":149,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.351355553Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58265"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.351547746Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2f3e57f4-ca18-4833-933d-fad8465844d6","remote_addr":"[fd00:1122:3344:101::c]:58265","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":159,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.478988875Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56096"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.528037658Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"da2aa236-1a55-44cd-b721-c0965f6bfe44","remote_addr":"[fd00:1122:3344:101::b]:56096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48990,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.528322642Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"3fd77e43-fdc4-4eab-b656-4339f563ad35","remote_addr":"[fd00:1122:3344:101::b]:56096","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.594749838Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64526"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.643563097Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4c0010e9-b91f-4b43-a3bc-0ea483eb0e87","remote_addr":"[fd00:1122:3344:101::c]:64526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48712,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:06.643837401Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"76fdfc10-72cf-49b7-9172-2f426b715a8d","remote_addr":"[fd00:1122:3344:101::c]:64526","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":96,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:18.778518196Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:18.778603927Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"4ccaed7b-0cc9-4eaf-ad4d-7c269e3c6966","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:47683","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:34.607977395Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:43599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:34.608165878Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2cba3436-17c5-40b2-ba27-88764279c5fb","remote_addr":"[fd00:1122:3344:101::a]:43599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":177,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:34.670894778Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:34.714533939Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"0e598e1c-a42b-4281-aeba-e1d7e94ac9ec","remote_addr":"[fd00:1122:3344:101::a]:35475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43577,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:34.714777482Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ba9db4cb-a56f-48a5-a097-77721afdabd0","remote_addr":"[fd00:1122:3344:101::a]:35475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.065706883Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43131"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.065913956Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"f62a4f2a-715d-49cb-9286-ef2f081e1094","remote_addr":"[fd00:1122:3344:101::b]:43131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":183,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.091206559Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47281"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.091338831Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"97bfc275-6e08-4d7e-a554-056252160cf3","remote_addr":"[fd00:1122:3344:101::c]:47281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":74,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.169347722Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.215987268Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"9a722a07-34ad-4fcc-b625-5f34abfb3580","remote_addr":"[fd00:1122:3344:101::b]:65139","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46575,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.21610829Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.258666264Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d46da424-8ca4-4caf-a788-1af050e3a49b","remote_addr":"[fd00:1122:3344:101::c]:60824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42499,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.258893228Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"993766d3-ed1b-4b1a-a7e1-e2142ac6097a","remote_addr":"[fd00:1122:3344:101::b]:65139","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":116,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:36.259147212Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"794cb0bb-928b-4b0e-a20f-5f66744ffc86","remote_addr":"[fd00:1122:3344:101::c]:60824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":162,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:48.810671899Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55005"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:00:48.810761211Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d9c128c2-fe5b-4804-bacb-3a85de07b084","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:55005","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":67,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:04.568635886Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59267"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:04.568834939Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"15ff7aba-1b8f-44a8-a581-661b00fc8385","remote_addr":"[fd00:1122:3344:101::a]:59267","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:04.802626259Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44239"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:04.846371991Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"81e71f7a-f11b-4533-bb0f-ccad60921136","remote_addr":"[fd00:1122:3344:101::a]:44239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43697,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:04.846636585Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fd388a61-85bf-4111-9dda-a252b7dfe319","remote_addr":"[fd00:1122:3344:101::a]:44239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.163363238Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56449"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.163588141Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"745599af-da99-4d5e-804a-23ef66e166c7","remote_addr":"[fd00:1122:3344:101::b]:56449","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":203,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.378424703Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.378623076Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"562acb88-6096-4d5a-aac4-1c8d6ec06971","remote_addr":"[fd00:1122:3344:101::c]:47624","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.41917649Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56485"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.468958284Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"ca311714-4d45-43cc-972f-1ab2517aca9a","remote_addr":"[fd00:1122:3344:101::b]:56485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49672,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.469200827Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"fc29714b-e4a1-4eb7-8845-3ad60ba64bbe","remote_addr":"[fd00:1122:3344:101::b]:56485","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.576552292Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.623886539Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"24cf40dc-47eb-414b-8f72-b81d8ba74e6a","remote_addr":"[fd00:1122:3344:101::c]:37600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47227,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:06.624676321Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"a04a479d-e7b3-40ed-8fc9-fa09b0570874","remote_addr":"[fd00:1122:3344:101::c]:37600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":642,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:18.842583338Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63551"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:18.842677549Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"1c5406fa-9081-43b7-a71d-4966e79737a9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:63551","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:25.297228699Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:49639"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:25.308111834Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"d969bff5-da17-4d52-8f61-307c7655b9c1","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":10379,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:34.636368137Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:34.63656773Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"38f237c3-665c-4937-a739-d0038d8cd8d4","remote_addr":"[fd00:1122:3344:101::a]:56763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:34.7396309Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:34.784876465Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"88a1fe34-6a87-4349-8ff2-254e2d2ecff1","remote_addr":"[fd00:1122:3344:101::a]:46451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45195,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:34.785171609Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"50e37df2-7f80-4d7d-b6d2-6f59e2af780a","remote_addr":"[fd00:1122:3344:101::a]:46451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:35.977244775Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50642"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:35.977470989Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"949008cd-1a29-4ca0-951b-e3c6c4ec3237","remote_addr":"[fd00:1122:3344:101::b]:50642","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":205,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.069585003Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:61619"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.069774546Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8c7bbf30-57c7-4d4b-b4a5-95696869f11d","remote_addr":"[fd00:1122:3344:101::c]:61619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":161,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.09711664Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48982"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.149015446Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"088564fe-1f9f-4b46-bb65-faeee71d6a1b","remote_addr":"[fd00:1122:3344:101::b]:48982","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51829,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.149345551Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"8622b076-353e-4317-8fde-36f44feaa47f","remote_addr":"[fd00:1122:3344:101::b]:48982","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.180132207Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48821"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.226031382Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"4055a300-51e1-43f1-a67b-4f10344eb291","remote_addr":"[fd00:1122:3344:101::c]:48821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45803,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:36.226299306Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"4370497f-f9fe-4587-80ca-f92e92bda2da","remote_addr":"[fd00:1122:3344:101::c]:48821","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:48.874264223Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33853"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:48.875177097Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"08dfeb08-8806-4486-9807-038098cc97a1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33853","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":771,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:01:55.296372191Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"6e346e5a-973b-4b48-9dff-f61df692682b","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":446,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:04.521069107Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:33194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:04.52126837Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2f7f2dc4-65b5-4b34-8f76-c48ad65c8088","remote_addr":"[fd00:1122:3344:101::a]:33194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:04.878199233Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59900"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:04.925084783Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"88e4013f-f67a-4789-b96d-4bcea415138e","remote_addr":"[fd00:1122:3344:101::a]:59900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46822,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:04.925330267Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"0b3eead5-a1b7-49ea-8ccc-07329c591e3e","remote_addr":"[fd00:1122:3344:101::a]:59900","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.03731687Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44000"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.03864494Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d2a2cdee-3db3-48c5-ae69-a0ef2822b80e","remote_addr":"[fd00:1122:3344:101::b]:44000","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1301,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.314894562Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39998"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.315046614Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"c207378c-e7da-4072-85c3-28d714f3a447","remote_addr":"[fd00:1122:3344:101::c]:39998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.477068597Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.524591546Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"86e5d52b-f9ea-4214-a133-45ba88f6e6bb","remote_addr":"[fd00:1122:3344:101::c]:50599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47436,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.524739608Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52556"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.533706494Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"46436098-14fd-482b-830a-4adccf8b0157","remote_addr":"[fd00:1122:3344:101::c]:50599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8695,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.577419946Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"6b908b3c-f725-4d83-846b-c9646e66f778","remote_addr":"[fd00:1122:3344:101::b]:52556","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52570,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:06.577844852Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"90dfe442-7907-4cf7-99e9-82346537fc6c","remote_addr":"[fd00:1122:3344:101::b]:52556","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:18.90824256Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:18.908356851Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"543b2e93-5afc-45be-bc5b-c0c5bc697912","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64690","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:25.297667332Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"11ed1c89-0d10-4ec9-9a14-7493be326870","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":956,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:34.495781234Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45383"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:34.495981657Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"d22aa9e5-fc93-461e-8c21-309a5f18cd4d","remote_addr":"[fd00:1122:3344:101::a]:45383","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:34.673544975Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34880"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:34.715586221Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"c2157896-4c13-4489-87a9-8b011219ee00","remote_addr":"[fd00:1122:3344:101::a]:34880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":41990,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:34.716564686Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"1ed56862-3fc1-4fcf-b1eb-9a9c8356f3b7","remote_addr":"[fd00:1122:3344:101::a]:34880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":800,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.000628134Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52698"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.000837118Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"05ff1ba9-fd31-4574-b643-0328f97a34fa","remote_addr":"[fd00:1122:3344:101::b]:52698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":195,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.168526876Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57801"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.168701519Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"237ffd14-8802-46c6-a3e2-81df8557f511","remote_addr":"[fd00:1122:3344:101::c]:57801","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.211312894Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.260090572Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"6ab6e115-c138-499f-94d4-1efee633ca56","remote_addr":"[fd00:1122:3344:101::b]:34355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48719,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.260265915Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41531"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.264017922Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c35bc418-ad0c-4d62-94af-3061571ec076","remote_addr":"[fd00:1122:3344:101::b]:34355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3592,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.304731837Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d69bf7c2-ea1f-4f1c-88c7-272eb485e7bf","remote_addr":"[fd00:1122:3344:101::c]:41531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44384,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:36.305066883Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"18358635-f0ef-4c47-972f-b6d727859545","remote_addr":"[fd00:1122:3344:101::c]:41531","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":143,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:48.94066922Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60037"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:48.940763732Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d142d161-43a0-4b90-89ed-063314239089","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60037","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":90,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:02:55.297157795Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"58596048-829c-45cc-8a46-769758739940","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:04.651737754Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:42382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:04.651928787Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2b18ba2c-b89e-4ee7-8924-0d5fea44417e","remote_addr":"[fd00:1122:3344:101::a]:42382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:04.825295022Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:04.871115195Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"aeb110a8-1da0-4802-b25a-7f24cb0a33ad","remote_addr":"[fd00:1122:3344:101::a]:55852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45772,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:04.871365719Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ef9e6f16-3d33-46b7-953a-45014d4ff646","remote_addr":"[fd00:1122:3344:101::a]:55852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.050273856Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:63070"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.050491589Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"65e9afdb-1f05-4e7e-b0e9-f689fdceb2d9","remote_addr":"[fd00:1122:3344:101::b]:63070","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":198,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.300572504Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36669"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.301474068Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"3d532ff1-fe61-4314-b024-3a2c21f2960b","remote_addr":"[fd00:1122:3344:101::c]:36669","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":881,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.36636769Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.41523414Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e072afd3-fcb3-463d-872b-a65acc111c1e","remote_addr":"[fd00:1122:3344:101::b]:53582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48804,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.415493514Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"257563c3-b69e-4df5-af25-e14fc41e72b7","remote_addr":"[fd00:1122:3344:101::b]:53582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.448167459Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47053"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.491486194Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"e743a636-2925-45af-a5d1-101da88a03c5","remote_addr":"[fd00:1122:3344:101::c]:47053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43244,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:06.491798269Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"ce261a8e-e38b-4c4c-8720-71ebd8dae743","remote_addr":"[fd00:1122:3344:101::c]:47053","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:18.973784841Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50709"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:18.973865293Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"f74754e4-0bc0-4d98-9af6-844c053bc738","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:50709","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":73,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:25.310217681Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"5ceaf010-febd-471d-b233-891c57c70a07","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1745,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:34.585085704Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53179"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:34.585295688Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"770274b4-c370-4e60-99e2-62c8d8d7fb96","remote_addr":"[fd00:1122:3344:101::a]:53179","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:34.730449135Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62390"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:34.777084711Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"36405a2f-696c-410f-bd09-4bca419be816","remote_addr":"[fd00:1122:3344:101::a]:62390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46595,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:34.777521848Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"f72e7a4f-9b11-47b8-afc9-a642e9717208","remote_addr":"[fd00:1122:3344:101::a]:62390","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":187,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.137595626Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37643"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.13781422Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"2e9c90b1-eff2-47bb-b0f2-62ec6dacdebc","remote_addr":"[fd00:1122:3344:101::c]:37643","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":199,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.178539736Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.178699089Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"51857a91-2056-44bd-b55d-41cd6bcc7d9d","remote_addr":"[fd00:1122:3344:101::b]:47408","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.221110501Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.267450292Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d7ec03ec-7af1-4bb8-9545-47d3f34a889a","remote_addr":"[fd00:1122:3344:101::c]:54742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46282,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.267706246Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"65b5b355-ff58-4684-a027-f70cd230d249","remote_addr":"[fd00:1122:3344:101::c]:54742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.352825214Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51852"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.400214632Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"291a131b-1b4d-4e0f-8473-cb1f4795eba1","remote_addr":"[fd00:1122:3344:101::b]:51852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47286,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:36.400579597Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"d1dc9582-55f2-4ddc-aeb1-a571dd52436b","remote_addr":"[fd00:1122:3344:101::b]:51852","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:49.005533911Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58078"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:49.005680263Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"292efb68-dd61-4cb2-9d6f-e5a958639f53","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58078","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:03:55.351004048Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"07ce645e-1dcc-44dd-863d-e9518dae9346","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":567,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.805332597Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55486"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.80549429Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"564fc20c-7b96-47e4-96f8-f65c0da67c06","remote_addr":"[fd00:1122:3344:101::a]:55486","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.841864781Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55959"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842052733Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842094424Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842127885Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842135895Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842170045Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842187986Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842201136Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.842217016Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"904337a0-65e7-4b16-91cd-64ed784440f4","remote_addr":"[fd00:1122:3344:101::a]:55959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":337,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:04.972485828Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39817"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:05.02017048Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"18a9a5b3-dfb5-4e9f-b221-b0bf911976c6","remote_addr":"[fd00:1122:3344:101::a]:39817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47606,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:05.020565986Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c669a414-4edd-4f74-83c7-7f9e664accc6","remote_addr":"[fd00:1122:3344:101::a]:39817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.181452589Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:44435"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.182592807Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"7a7b2d80-468c-41d9-a199-daf085fa6331","remote_addr":"[fd00:1122:3344:101::c]:44435","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.1827782Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38166"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.18345254Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.18349023Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.183502431Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.18350968Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.183521471Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.183541431Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.183574952Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.183588792Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"14f9c44c-ff65-4755-91ed-8c7f356ad6c0","remote_addr":"[fd00:1122:3344:101::b]:38166","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":273,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.194182902Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40715"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.194315884Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"1bf3fbf4-8a03-4558-93c3-a0ca31fd9fd4","remote_addr":"[fd00:1122:3344:101::b]:40715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.401705603Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52766"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.401988198Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: beaa816a-f113-442c-98ce-abba012f2963, kind: Service { id: d722cbf1-822b-421d-89f9-027b7d805ad2 }, name: Name(\"external-dns-d722cbf1-822b-421d-89f9-027b7d805ad2\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 135, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf6bd4f6-a4d7-4e9c-bfd1-6407be67c63e, kind: Service { id: 244b3e14-8cd3-4efe-a576-185d13c0ab68 }, name: Name(\"external-dns-244b3e14-8cd3-4efe-a576-185d13c0ab68\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 199, 166])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 3c88d731-6d88-459a-bd89-36c74cc5b129, kind: Service { id: 59cebf87-1ddd-4cf3-8dfe-af47c91c765f }, name: Name(\"nexus-59cebf87-1ddd-4cf3-8dfe-af47c91c765f\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 93])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 4fca6926-e254-4dac-99da-bc3d4e80a3c6, kind: Service { id: 88d03fa6-91e4-44cd-bd2f-3ca495b76e2c }, name: Name(\"nexus-88d03fa6-91e4-44cd-bd2f-3ca495b76e2c\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 148, 170])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: bf0c2746-7bc1-42c8-9696-562773c31e0d, kind: Service { id: b7f02198-e503-4542-8190-5c4bb0e37c4d }, name: Name(\"nexus-b7f02198-e503-4542-8190-5c4bb0e37c4d\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 184, 113])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.402040548Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.402079719Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.402092139Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.402116819Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.40213313Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.40215057Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.40217009Z","hostname":"sock","pid":1562,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"256fcf75-6271-4398-ab7f-064fd94bc3ff","remote_addr":"[fd00:1122:3344:101::c]:52766","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":433,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.469814164Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:54726"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.514899047Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"bb1c3518-e5b2-4ab0-9011-2558a6e042b9","remote_addr":"[fd00:1122:3344:101::b]:54726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45003,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.515021849Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33312"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.518702194Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"638607c7-6233-483e-bb05-af6886bdf4d7","remote_addr":"[fd00:1122:3344:101::b]:54726","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3459,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.563498892Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"5f019009-baaf-43a7-9637-93d79f3247d3","remote_addr":"[fd00:1122:3344:101::c]:33312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48384,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:06.563910809Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"04a561f3-28ac-4d02-8c54-8a93d22766bd","remote_addr":"[fd00:1122:3344:101::c]:33312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:19.038264895Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53672"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:19.039009697Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"d88b5304-a251-45b8-a0bb-025ea4ae0d5b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53672","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":726,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:25.351576676Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"27202090-a12f-447f-8739-adcd43cda1fd","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":733,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:34.500082886Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64379"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:34.500264878Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"69e4fe3a-03f3-46bb-a686-41d35919a4a6","remote_addr":"[fd00:1122:3344:101::a]:64379","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":165,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:34.695197779Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:34.740430884Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"722cebd0-f7ce-428a-a0a1-75984b01fbe6","remote_addr":"[fd00:1122:3344:101::a]:44950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45176,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:34.740673528Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"5cf45305-1535-45fc-8da9-d855fc7e30ec","remote_addr":"[fd00:1122:3344:101::a]:44950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.037467679Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.038432253Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"b70d7701-aeae-40f5-aaf9-6d943a79e9e8","remote_addr":"[fd00:1122:3344:101::b]:47820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":942,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.059576663Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49590"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.059720685Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"18221e8f-c557-4052-840b-49ba5a682bb4","remote_addr":"[fd00:1122:3344:101::c]:49590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.127589743Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.1756015Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"290a098a-9ce5-4d59-9494-bdebe8bd93d9","remote_addr":"[fd00:1122:3344:101::b]:38009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47947,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.175698051Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:53969"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.220210655Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"1274ba6c-9462-426b-9736-dd20c8d178ec","remote_addr":"[fd00:1122:3344:101::c]:53969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44455,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.220452979Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"c7dfa368-fad5-4ee2-a524-cded08fec3da","remote_addr":"[fd00:1122:3344:101::b]:38009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:36.221280191Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"04ad8db7-2634-4560-b83c-99f076ee7371","remote_addr":"[fd00:1122:3344:101::c]:53969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":727,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:49.073158252Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:49.073249454Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"a176498d-b9fe-497f-be30-f32853d1c42e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38018","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":64,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:52.475237413Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:52.475297203Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:52.512526797Z","hostname":"sock","pid":1562,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999711s","next_cleanup":"Instant { tv_sec: 2099, tv_nsec: 22593662 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:04:55.35126211Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"ee8ef8a9-0492-4a1a-a1b6-c5157d3b3e02","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":551,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:04.53084559Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:59362"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:04.531017952Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"0e09ecf8-0585-4313-9337-44e5ccf4ab6b","remote_addr":"[fd00:1122:3344:101::a]:59362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:04.919332731Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58396"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:04.962952581Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fd6a9501-12d7-4c60-a928-c0a6213e3992","remote_addr":"[fd00:1122:3344:101::a]:58396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43562,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:04.963217065Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"9c1917d6-8fce-4f83-9ec3-38899a106d26","remote_addr":"[fd00:1122:3344:101::a]:58396","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.056260582Z","hostname":"sock","pid":1562,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.088948486Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.08917253Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"cd537b57-7e71-4db7-af35-133fce019df0","remote_addr":"[fd00:1122:3344:101::b]:51586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":171,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.149585695Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51428"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.149789758Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"8dc2620a-ae81-49ab-8eb4-807959627334","remote_addr":"[fd00:1122:3344:101::c]:51428","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.341243865Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.388681914Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d7a3b48a-0b82-4824-8bb3-1e2d36120f27","remote_addr":"[fd00:1122:3344:101::c]:64151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47374,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.388803585Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:37790"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.435533603Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"fe85802c-0aaf-4b6d-b575-7a0dca08082b","remote_addr":"[fd00:1122:3344:101::b]:37790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46664,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.435751436Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"608b3db0-5abf-4e52-a4b5-38f2b0bdec9d","remote_addr":"[fd00:1122:3344:101::c]:64151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:06.436572409Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"45bf863b-8363-4839-a839-d03ea2030cda","remote_addr":"[fd00:1122:3344:101::b]:37790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":743,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:19.104906832Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:19.104996613Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"3b6530b3-d8ae-45b0-9a15-ce9516b2e0d1","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:60650","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":72,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:25.350973454Z","hostname":"sock","pid":1562,"uri":"/5afeac69-ea1f-40c0-9e56-34defe323774","method":"GET","req_id":"6ddf8fcf-46ab-476c-ba35-1f2a4b894197","remote_addr":"[fd00:1122:3344:101::d]:49639","local_addr":"[fd00:1122:3344:101::1]:48051","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"5afeac69-ea1f-40c0-9e56-34defe323774","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":522,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.377373623Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44035"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.377567536Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"01ae330c-9668-41f9-974a-1826e14ff369","remote_addr":"[fd00:1122:3344:101::a]:44035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":178,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.37914284Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9359447) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.37916269Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"PUT","req_id":"470c4fc8-c419-4ac0-a7c8-4e9fa6622337","remote_addr":"[fd00:1122:3344:101::a]:44035","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1388,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.530140816Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36980"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.536811817Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:548","metadata":"InstanceMetadata { silo_id: fbab6021-23c8-40a2-80cb-415e072de604, project_id: e1946849-8155-4f60-81fd-4592ebf4e3a1 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-22T02:05:30.113046Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(9359447))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6bc748eae37\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: fb2837c3-a8bb-4603-900d-679019589196, read_only_parent: Some(Volume { block_size: 512, id: fe5e1981-8f05-477f-b900-39f1e6cfcdb8, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 64ed125e-72fb-47fc-b334-2ffb7cdc64ec, key: Some(\"Z7X/gedeJjEGkOWAXZaN7OXTDufhx/LfXx7h60PCRmw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: fb2837c3-a8bb-4603-900d-679019589196, key: Some(\"qTh+nVaPItVlnLPrKIijrmDQ5LcSlD8XlskGkGBVc/8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAxrxDWWNZYAACvENZYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAMa8Q1ljWWAAArxDWWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiZWI0NTkwZS0zNmMyLTQwMzMtOWNmZS1iMmM3NjAxNGM2ZTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRGh0ek5ETUd2MlR2WlJuZStFdk1DYjJGQzk3STRvRjAybmg1dm9rcmlSbiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.536899768Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:582","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.538218568Z","hostname":"sock","pid":1562,"component":"InstanceManager","file":"sled-agent/src/instance.rs:942","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(9359447))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: f9707ec3-bd9b-4a2b-abab-965fb3039320, kind: Instance { id: beb4590e-36c2-4033-9cfe-b2c76014c6e2 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9359447), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-6bc748eae37\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: fb2837c3-a8bb-4603-900d-679019589196, read_only_parent: Some(Volume { block_size: 512, id: fe5e1981-8f05-477f-b900-39f1e6cfcdb8, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 64ed125e-72fb-47fc-b334-2ffb7cdc64ec, key: Some(\"Z7X/gedeJjEGkOWAXZaN7OXTDufhx/LfXx7h60PCRmw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: fb2837c3-a8bb-4603-900d-679019589196, key: Some(\"qTh+nVaPItVlnLPrKIijrmDQ5LcSlD8XlskGkGBVc/8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAxrxDWWNZYAACvENZYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAMa8Q1ljWWAAArxDWWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiZWI0NTkwZS0zNmMyLTQwMzMtOWNmZS1iMmM3NjAxNGM2ZTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRGh0ek5ETUd2MlR2WlJuZStFdk1DYjJGQzk3STRvRjAybmg1dm9rcmlSbiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-22T02:05:30.113046Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.53833768Z","hostname":"sock","pid":1562,"uri":"/instances/beb4590e-36c2-4033-9cfe-b2c76014c6e2","method":"PUT","req_id":"531ab209-1d50-472e-ad22-045b7fa66046","remote_addr":"[fd00:1122:3344:101::a]:36980","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.624562155Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:40427"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.626290182Z","hostname":"sock","pid":1562,"uri":"/instances/beb4590e-36c2-4033-9cfe-b2c76014c6e2/state","method":"PUT","req_id":"e8336e70-d632-40dd-b385-1bf9bbcc5ba3","remote_addr":"[fd00:1122:3344:101::a]:40427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1662,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.648356276Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 246, 17, 154]), slot: 0, vni: Vni { inner: 9359447 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.649068106Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.681482937Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:30.703078384Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:32.095118767Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:32.095362521Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 73.962772ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:32.274735706Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 54.407886ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:32.361045622Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 292.694103ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:32.431545739Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 354.642448ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:32.740681339Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 649.734535ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:33.110593569Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 921.979368ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:33.772237325Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:34.478383434Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44427"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:34.492897854Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:39824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:34.493078627Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"31f5eaca-272a-4250-8d58-0eaea45d8322","remote_addr":"[fd00:1122:3344:101::a]:39824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:34.620736859Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37773"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:34.667490517Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"8c89f124-a771-489a-896b-2e2b3e0932d8","remote_addr":"[fd00:1122:3344:101::a]:37773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46710,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:34.668473192Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"2788930f-1823-4f47-bb55-42254f0952cd","remote_addr":"[fd00:1122:3344:101::a]:37773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":848,"response_code":"200"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 973.910784ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:34.704161122Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 1.385389594s","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:35.687343096Z","hostname":"sock","pid":1562,"zone":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:35.986736538Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47767"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.018966956Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42616"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.019804159Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"1b417a00-eb06-4770-b49c-8173f69f4a8e","remote_addr":"[fd00:1122:3344:101::b]:42616","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":171,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.074990504Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58854"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.082260764Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57245"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.083252029Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"bb20eac9-269f-4560-ae35-5fec23f37dc8","remote_addr":"[fd00:1122:3344:101::c]:57245","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":948,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.132340692Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:61386"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.177522156Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"50aaf953-bb1c-488b-9a19-e00bb8d8d032","remote_addr":"[fd00:1122:3344:101::b]:61386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45120,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.17778722Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"7b67bcb2-9b4c-4fdd-907b-22a0fbcd82aa","remote_addr":"[fd00:1122:3344:101::b]:61386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.279948497Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:65077"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.326827206Z","hostname":"sock","pid":1562,"uri":"/switch-ports","method":"POST","req_id":"d340f419-1fd0-4cd6-b86d-bb57cb6a1070","remote_addr":"[fd00:1122:3344:101::c]:65077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46791,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:36.32709566Z","hostname":"sock","pid":1562,"uri":"/network-bootstore-config","method":"GET","req_id":"e165fa97-b0b0-42f9-a2db-15a4732a933d","remote_addr":"[fd00:1122:3344:101::c]:65077","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":98,"response_code":"200"} {"msg":"Started propolis in zone: oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:37.100585589Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1427"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 35.097858ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:37.109099478Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 147.778453ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:37.154200011Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 131.257217ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:37.31203267Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\") failed: Property not found. retry in 438.140932ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:37.454570928Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:37.903710867Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1436"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:37.937974766Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:1450"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAxrxDWWNZYAACvENZYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAMa8Q1ljWWAAArxDWWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJiZWI0NTkwZS0zNmMyLTQwMzMtOWNmZS1iMmM3NjAxNGM2ZTIiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJRGh0ek5ETUd2MlR2WlJuZStFdk1DYjJGQzk3STRvRjAybmg1dm9rcmlSbiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-6bc748eae37\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: fb2837c3-a8bb-4603-900d-679019589196, read_only_parent: Some(Volume { block_size: 512, id: fe5e1981-8f05-477f-b900-39f1e6cfcdb8, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 64ed125e-72fb-47fc-b334-2ffb7cdc64ec, key: Some(\"Z7X/gedeJjEGkOWAXZaN7OXTDufhx/LfXx7h60PCRmw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19002\", \"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: fb2837c3-a8bb-4603-900d-679019589196, key: Some(\"qTh+nVaPItVlnLPrKIijrmDQ5LcSlD8XlskGkGBVc/8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::15]:19001\", \"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::18]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: beb4590e-36c2-4033-9cfe-b2c76014c6e2, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: e1946849-8155-4f60-81fd-4592ebf4e3a1, silo_id: fbab6021-23c8-40a2-80cb-415e072de604 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:37.945158964Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:689"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.110142812Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:691"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.110833472Z","hostname":"sock","pid":1562,"uri":"/instances/beb4590e-36c2-4033-9cfe-b2c76014c6e2/state","method":"GET","req_id":"165d80a4-5153-4830-bd52-4bb37ff8484f","remote_addr":"[fd00:1122:3344:101::a]:44427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3632396,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-06-22T02:05:38.110841772Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.110899123Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.110907593Z","hostname":"sock","pid":1562,"uri":"/instances/beb4590e-36c2-4033-9cfe-b2c76014c6e2/state","method":"GET","req_id":"af5c1d5e-a034-4517-b131-2ebdb72e0d65","remote_addr":"[fd00:1122:3344:101::c]:58854","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2035828,"response_code":"200"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.110941014Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-22T02:05:38.110841772Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }","propolis_id":"7f72e69d-c5e6-46d4-b155-157044006c8f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.110969725Z","hostname":"sock","pid":1562,"uri":"/instances/beb4590e-36c2-4033-9cfe-b2c76014c6e2/state","method":"GET","req_id":"e8330d2d-e439-40f6-a220-d4f32e4052f4","remote_addr":"[fd00:1122:3344:101::b]:47767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2124139,"response_code":"200"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.111722756Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }, propolis_id: 7f72e69d-c5e6-46d4-b155-157044006c8f (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-22T02:05:38.110841772Z }, migration_state: None }","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-06-22T02:05:38.142306569Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.142327349Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.142342859Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-22T02:05:38.142306569Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }","propolis_id":"7f72e69d-c5e6-46d4-b155-157044006c8f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:38.14235466Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }, propolis_id: 7f72e69d-c5e6-46d4-b155-157044006c8f (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-22T02:05:38.142306569Z }, migration_state: None }","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.750357645Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37670"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.751062945Z","hostname":"sock","pid":1562,"uri":"/instances/beb4590e-36c2-4033-9cfe-b2c76014c6e2/state","method":"PUT","req_id":"79953577-e237-44fd-8ead-aa6f080295fb","remote_addr":"[fd00:1122:3344:101::a]:37670","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":634,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-06-22T02:05:47.751177477Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.751193758Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.751209618Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-22T02:05:47.751177477Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }","propolis_id":"7f72e69d-c5e6-46d4-b155-157044006c8f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.751234648Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(7f72e69d-c5e6-46d4-b155-157044006c8f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-22T02:05:30.165147351Z }, propolis_id: 7f72e69d-c5e6-46d4-b155-157044006c8f (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-22T02:05:47.751177477Z }, migration_state: None }","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-06-22T02:05:47.797235824Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.797266055Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.797299656Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-06-22T02:05:47.797235824Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-22T02:05:47.797235824Z }","propolis_id":"7f72e69d-c5e6-46d4-b155-157044006c8f"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.797324236Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:610","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.797329766Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.79760222Z","hostname":"sock","pid":1562,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:47.797677441Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/ad11261d-76fd-447c-b4fa-06153786f4ac/debug/bundle/zone\", \"/pool/int/dfa37dd5-5aca-43c5-b94a-79f447e6cd21/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/fbfdf456-46af-4ca5-8e50-42e804e95de3/crypt/debug\", \"/pool/ext/072ad6a6-48d3-47f5-b5ad-907cb6ccb208/crypt/debug\", \"/pool/ext/21f0441c-e835-4662-89b4-8f9c50d54833/crypt/debug\", \"/pool/ext/ce3883c7-6e59-4796-9a58-b1fefe1b0724/crypt/debug\", \"/pool/ext/cadffa03-0f46-493d-807b-355e3bd965dd/crypt/debug\", \"/pool/ext/a5bbfc86-7c7e-4dd8-823e-791e9cd96508/crypt/debug\", \"/pool/ext/37a51968-6b4b-4243-8e26-612da188294b/crypt/debug\", \"/pool/ext/31de0ecb-89b0-4d10-a4da-a6cbbdd4101a/crypt/debug\", \"/pool/ext/9f456447-b67b-4176-b7d7-23a75139f1da/crypt/debug\"] }","zone_name":"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:48.454288841Z","hostname":"sock","pid":1562,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f\", bundle_id: d6bc690a-05f5-47b8-a669-aff9c4585041 }, time_created: 2024-06-22T02:05:47.797976786Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_7f72e69d-c5e6-46d4-b155-157044006c8f","v":0,"name":"SledAgent","level":40,"time":"2024-06-22T02:05:48.454372532Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:784"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:49.136307536Z","hostname":"sock","pid":1562,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:49.136471218Z","hostname":"sock","pid":1562,"uri":"/baseboard","method":"GET","req_id":"93dac6d9-e95b-4ae7-ab27-da6cb236f3e6","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59075","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":124,"response_code":"200"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.185403637Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.211939508Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-22T02:05:47.797235824Z }, propolis_id: 7f72e69d-c5e6-46d4-b155-157044006c8f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-22T02:05:50.211868007Z }, migration_state: None }","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.313026848Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34959"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.313241701Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"f95772ef-11e7-4766-9a39-a2e8c6470e30","remote_addr":"[fd00:1122:3344:101::a]:34959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":198,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.313479955Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9359447) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.313499915Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"PUT","req_id":"d5fa04b2-7156-4988-92bf-ef5dd0598267","remote_addr":"[fd00:1122:3344:101::a]:34959","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":102,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.507905058Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-22T02:05:47.797235824Z }, propolis_id: 7f72e69d-c5e6-46d4-b155-157044006c8f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-22T02:05:50.211868007Z }, migration_state: None }","instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.512138962Z","hostname":"sock","pid":1562,"instance_id":"beb4590e-36c2-4033-9cfe-b2c76014c6e2","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.705849595Z","hostname":"sock","pid":1562,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44861"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.706059768Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"GET","req_id":"54f3f14c-8161-4b63-a903-aa5388edb1d6","remote_addr":"[fd00:1122:3344:101::a]:44861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":172,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.706303432Z","hostname":"sock","pid":1562,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 246, 17, 154])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9359447) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-22T02:05:50.706502065Z","hostname":"sock","pid":1562,"uri":"/v2p","method":"DELETE","req_id":"e49ec18d-4623-46a5-9c40-36786634cdad","remote_addr":"[fd00:1122:3344:101::a]:44861","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":252,"response_code":"204"}