[ Jun 21 21:37:11 Enabled. ] [ Jun 21 21:37:12 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jun 21 21:37:12 Method "start" exited with status 0. ] [ Jun 21 21:37:12 Rereading configuration. ] [ Jun 21 21:37:12 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.027268477Z","hostname":"sock","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.027444181Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.080589185Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.080633156Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.194864452Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.203584775Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.211688866Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.813755579Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:a02a::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.955204345Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.955233206Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.955333738Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.955418379Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.95542766Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.9554472Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"sock\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958220545Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:12.958227515Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958240145Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958243755Z","hostname":"sock","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958267566Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958331787Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958374898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958396398Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.958407599Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.969757953Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:12.969794394Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.069367549Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.274656272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.274695553Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.274701293Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.274706253Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.274716703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.274722593Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.274739173Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.283501507Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.283530798Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.369009493Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576182353Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576229644Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576255215Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576267545Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576279595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576284975Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576292845Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576317626Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576330396Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576336496Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576341196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576378057Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576412898Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576428488Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576432868Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576455889Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576461239Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576467339Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576471549Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576475699Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576491449Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.57649612Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.5765015Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.57650562Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.57651069Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.57651859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.57652252Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.57652757Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576553641Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576559321Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576563461Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576569081Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576585621Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576591121Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576596431Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576600342Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576605952Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576621272Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576626102Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576653043Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576659553Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576664733Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576668653Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576676023Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576693863Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576699084Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576703804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576710224Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576715484Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576720154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576732014Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576754354Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576759685Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576764625Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576768495Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576779325Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576783435Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576815656Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576821196Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576825426Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576829526Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576865477Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576877587Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576885907Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576902708Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576908068Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576912618Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576937388Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.576942288Z","hostname":"sock","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.587994548Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:710"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.588018758Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999884367s","next_cleanup":"Instant { tv_sec: 5194, tv_nsec: 988841726 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.588097379Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3547"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.58811861Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.588184881Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.588196281Z","hostname":"sock","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591358084Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591431936Z","hostname":"sock","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.2.164,\n last: 10.151.2.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.2.164,\n 10.151.2.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.59165436Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.59166656Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591728362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591774972Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591792303Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591798083Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591830834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591845984Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591854404Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591872634Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591877125Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:a02a::1","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591881255Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591890115Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.591933246Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592292933Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592421635Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592429545Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592465566Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592470436Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592606869Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592727422Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592767512Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: a27c09e4-c9a4-4472-b248-8c2ae9bae00f, rack_id: 884d2112-d3cd-48eb-9cc7-ecb0a29f78a9, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592871444Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:43025","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: a27c09e4-c9a4-4472-b248-8c2ae9bae00f, rack_id: 884d2112-d3cd-48eb-9cc7-ecb0a29f78a9, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592959746Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592969446Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592989297Z","hostname":"sock","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.592995847Z","hostname":"sock","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.593005177Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.593018017Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.593022017Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.593076188Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.593090339Z","hostname":"sock","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.593097899Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.593103349Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.595760062Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3779"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.605469804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.616174456Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.625337548Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.634363817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.642921267Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.650572169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.659634739Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.667893662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.676363341Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.676411042Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.676415812Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.676420462Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.676424992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.676437822Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.682633425Z","hostname":"sock","pid":1555,"sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.682690566Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.682699236Z","hostname":"sock","pid":1555,"sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.682799298Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 50.169527ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.687783047Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fcc56b15-8980-4a2f-879f-9c405d39883f\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:37:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fcc56b15-8980-4a2f-879f-9c405d39883f\" }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.696371827Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_9af8bc90-90a3-45f5-b12b-12fd9904a27e/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.698621552Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.713868374Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_9af8bc90-90a3-45f5-b12b-12fd9904a27e/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.722331133Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_9af8bc90-90a3-45f5-b12b-12fd9904a27e/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.722377463Z","hostname":"sock","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.727467534Z","hostname":"sock","pid":1555,"sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:381"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.727479775Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_9af8bc90-90a3-45f5-b12b-12fd9904a27e/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.727485325Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.739510583Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 72.419733ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.740004873Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef58e90e-1699-4e39-8ed2-16481d55b562\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:37:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef58e90e-1699-4e39-8ed2-16481d55b562\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.798989663Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.813065202Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 158.994459ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.813618993Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a71e2d0b-97e4-4a6f-9fee-8c4ac5ba407d\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:37:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a71e2d0b-97e4-4a6f-9fee-8c4ac5ba407d\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.973736009Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 185.989247ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:13.974312821Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"46c44140-8a9a-41e4-b3c0-2c4914d77b4c\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:37:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"46c44140-8a9a-41e4-b3c0-2c4914d77b4c\" }"} {"msg":"Mounting oxi_9af8bc90-90a3-45f5-b12b-12fd9904a27e/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.017830064Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.030921163Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.031026596Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.031053826Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.031078767Z","hostname":"sock","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.161244709Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 302.25226ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.161959263Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"38768002-2e98-4f44-99b0-9243fd62e02a\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:37:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"38768002-2e98-4f44-99b0-9243fd62e02a\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.465050756Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 329.669884ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.465969864Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4bdd07c6-2984-4105-9e90-e5c30511e0fa\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:37:13 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4bdd07c6-2984-4105-9e90-e5c30511e0fa\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.725550393Z","hostname":"sock","pid":1555,"sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.72641195Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.726431001Z","hostname":"sock","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.726491022Z","hostname":"sock","pid":1555,"file":"oximeter/producer/src/lib.rs:276","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:276","interval":"30s","address":"[fd00:1122:3344:101::1]:44452","producer_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:14.727675185Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"141.191171ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.772673968Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.772709579Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.77276881Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:893","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.77277944Z","hostname":"sock","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773455994Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773482544Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773515335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773534335Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773735909Z","hostname":"sock","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773884502Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773910583Z","hostname":"sock","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.2.174, infra_ip_last: 10.151.2.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.2.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.2.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773918493Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3547"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.773964664Z","hostname":"sock","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:a02a::1]:43025","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.949162789Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.949228971Z","hostname":"sock","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:a02a::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.949540367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.949647849Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.949662169Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.9497051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.94971584Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.94972206Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:14.950993735Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"671.029377ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:14.955324721Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.987111362Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40461"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.987440718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:14.98754853Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"beb9e2fc-4196-40cc-b2a0-5348475fa272","remote_addr":"[fd00:1122:3344:101::1]:40461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":262,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.024919752Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:41363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.025016944Z","hostname":"sock","pid":1555,"uri":"/sled-role","method":"GET","req_id":"6041313a-94c3-4a51-ba58-9d0f294dce29","remote_addr":"[fd00:1122:3344:101::1]:41363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":37,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.026786679Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.02683117Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.027307559Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.027584745Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:812"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 6cd081a2-283c-43a2-a300-b0899484b0ba, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool) }, OmicronPhysicalDiskConfig { id: 91eb1bb1-3bbf-4c49-884a-85d7a34c2137, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool) }, OmicronPhysicalDiskConfig { id: 83b1edcd-e5df-45c2-b7a3-53c029865fba, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool) }, OmicronPhysicalDiskConfig { id: 6799c5cc-a2dd-4cd8-8455-656a136c2ca0, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool) }, OmicronPhysicalDiskConfig { id: e7e5d64e-5ccb-4079-a9eb-dd436ad5ea7e, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool) }, OmicronPhysicalDiskConfig { id: 3fe012df-50e6-4b55-b0c8-2345a041579e, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool) }, OmicronPhysicalDiskConfig { id: ef750f3b-5c0e-43a6-985a-c9de50bba937, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool) }, OmicronPhysicalDiskConfig { id: 755c2842-c938-497b-b846-6141fa1bfdc4, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool) }, OmicronPhysicalDiskConfig { id: 7b8511c8-ea59-4a7f-bfe7-3d59a7200e97, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.058658611Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.058765023Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57235"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 6cd081a2-283c-43a2-a300-b0899484b0ba, pool_id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 91eb1bb1-3bbf-4c49-884a-85d7a34c2137, pool_id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 83b1edcd-e5df-45c2-b7a3-53c029865fba, pool_id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 6799c5cc-a2dd-4cd8-8455-656a136c2ca0, pool_id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: e7e5d64e-5ccb-4079-a9eb-dd436ad5ea7e, pool_id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 3fe012df-50e6-4b55-b0c8-2345a041579e, pool_id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: ef750f3b-5c0e-43a6-985a-c9de50bba937, pool_id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 755c2842-c938-497b-b846-6141fa1bfdc4, pool_id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 7b8511c8-ea59-4a7f-bfe7-3d59a7200e97, pool_id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.05909871Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.05911882Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.05912817Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.05913516Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.059206792Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.059215732Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.059223752Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.059230472Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.169206574Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:15.169268075Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"5b452253-5f6d-4617-91b5-3f263edef77d (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:15.69931661Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"918.594688ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:16.113635069Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.122893512Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.289969576Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.290093839Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.290101119Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.290118419Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.29012803Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.290220321Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.290227201Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:16.647321276Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.14468709s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:16.822058322Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:17.011703903Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Automatically destroying dataset: oxp_5b452253-5f6d-4617-91b5-3f263edef77d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:17.170576665Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:17.370604413Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:17.370689155Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 73.468193ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:17.811241334Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:18.391545326Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 77.02865ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:18.391628007Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 261.707535ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:19.002318081Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:19.003394453Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.147715751s"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:19.180147809Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:19.825536822Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 467.418799ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:19.825817747Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_5b452253-5f6d-4617-91b5-3f263edef77d","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.067222036Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.293417433Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.293475174Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.293487544Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.293512855Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.293521245Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.511687273Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:20.511732644Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"41b82b1f-5def-467c-bc86-5c87195bd788 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.662866709Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.889050296Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.889105957Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.889118647Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.889190409Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.889207719Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.889392533Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:21.889410093Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:22.640712446Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:22.640767768Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:22.640776968Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:384","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:384","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.549692053s"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:23.362697288Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_41b82b1f-5def-467c-bc86-5c87195bd788/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:23.830831515Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:24.309452609Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:24.309728745Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:24.539773388Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2233"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.025880541Z","hostname":"sock","pid":1555,"component":"VmmReservoirManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Looking for unencrypted datasets in oxp_41b82b1f-5def-467c-bc86-5c87195bd788","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.137693449Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.145886991Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.145972333Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.145981133Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.145988613Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.145994973Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.155800378Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.155851969Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.250467806Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.258757161Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.258811191Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.258819292Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.258828162Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.258838882Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.258982495Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.259022626Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.294477029Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_44b9c17b-6e37-45fe-b9da-6182ccbe59a4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.302219043Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:25.313889404Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_44b9c17b-6e37-45fe-b9da-6182ccbe59a4","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.426731873Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.434760742Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.434789373Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.434795323Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.434801703Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.434810873Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.444304031Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.444336212Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.534450119Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.543166143Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.543216333Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.543226914Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.543285675Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.543302835Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.543454928Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.543465728Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.581891281Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e6f1c1a1-ab63-45ad-a43f-153910e0494a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.590084933Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_e6f1c1a1-ab63-45ad-a43f-153910e0494a","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.712705026Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.720701274Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.720725185Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.720746075Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.720753025Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.720757015Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.730132532Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.730157272Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.824271629Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.832169725Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.832226517Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.832244387Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.832260657Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.832282538Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.832433641Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.832441131Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.867885304Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_8ce4d8fb-4017-46e3-ac55-3c40125a531b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.875683729Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_8ce4d8fb-4017-46e3-ac55-3c40125a531b","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:25.993497085Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.000372641Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.000408232Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.000416742Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.000424163Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.000440223Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.010312069Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.01035243Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.101543059Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.109426605Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.109470856Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.109477816Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.109509597Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.109523827Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.109627689Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.1096788Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:26.113531696Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.144539372Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_a0c5f5f0-1cc5-45cc-9434-17b4abf3b972/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.152469839Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_a0c5f5f0-1cc5-45cc-9434-17b4abf3b972","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.278235084Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.285973927Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.285999278Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.286015938Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.286021698Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.286033518Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.295848133Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.295882974Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"85a7fd4f-88d4-408c-a832-0d457152c72f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.392320917Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.40054295Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.400569451Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.400575931Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.400592411Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.400598871Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.400773975Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.400795835Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:26.405117701Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.437298469Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_85a7fd4f-88d4-408c-a832-0d457152c72f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.446336648Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_85a7fd4f-88d4-408c-a832-0d457152c72f","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.571266667Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.579312576Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.579355347Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.579363127Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.579368618Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.579374598Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.590136341Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.590185722Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.691199246Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.699509771Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.699554612Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.699572112Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.699598923Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.699655154Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.699779246Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.699787386Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.737501344Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_6b2e5449-891f-4dfd-8e2d-78189eabe443/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.745457682Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_6b2e5449-891f-4dfd-8e2d-78189eabe443","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.868800449Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.876576233Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.876619514Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.876631214Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.876639315Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.876646765Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.886305446Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.886347337Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.974908774Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.98379404Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.983839661Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.983847311Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.983859191Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.983886142Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.984003364Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:26.984011734Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.019682692Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.02767364Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_3ae04f9e-dd4d-4980-8e86-31c81824c907","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.150534858Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.158390644Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.158446234Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.158481145Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.158634958Z","hostname":"sock","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 9af8bc90-90a3-45f5-b12b-12fd9904a27e (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: 89c103d3-b501-4e1d-8d78-ad9e833f154c (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.158687959Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.158697339Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.158778391Z","hostname":"sock","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.159044557Z","hostname":"sock","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"570044f0-3af9-488c-a59d-118808de7c51","remote_addr":"[fd00:1122:3344:101::1]:57235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12100205,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.188841288Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.18896334Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.189180994Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.189190204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.235111906Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.235150196Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.284294111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.284333312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.334244042Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.334298573Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.334323454Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.334381235Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.379044281Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.379099062Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.379106162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:27.408916743Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.432779977Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.436767596Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.440671933Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.453470517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.458107569Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:27.461960326Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:28.111742625Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/a27c09e4-c9a4-4472-b248-8c2ae9bae00f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:37:28.411121444Z","hostname":"sock","pid":1555,"component":"ProbeManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:28.980622251Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:28.980750133Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:28.980942677Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:29.469472608Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:29.947657604Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Profile for oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:29.947783717Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:29.947824298Z","hostname":"sock","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 50.685439ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:29.955296286Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:29.955335357Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:29.959137922Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 119.184165ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.016898007Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 71.312941ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.083875946Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 56.596795ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.090442386Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 287.174673ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.145338525Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 75.885434ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.216177481Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 101.662273ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.245179486Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 189.317966ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.394799044Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 238.702361ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.412557336Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 351.500805ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.463058958Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 403.17341ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.599667748Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 281.386479ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.661937573Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 471.529661ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.823156842Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 763.456403ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:30.952217112Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 1.182839192s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:31.012155651Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 874.892563ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:31.303531141Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 502.779466ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:31.725091073Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 939.261504ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:32.187168619Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 1.402023492s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:32.204009994Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 745.206801ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:32.236741843Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:a02a::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:32.618984886Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2250"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:32.619011576Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 566.231968ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:32.991633838Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 739.967891ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:33.135315998Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fee5:dd9c","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:33.173411484Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2257"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:33.179232819Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2318"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:33.300982384Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2503"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:33.532323443Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2667"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 1.495729466s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:33.567745636Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 736.212747ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:33.615003664Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:33.71564445Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2382"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 1.094148046s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:33.884073691Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.006437408Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2417"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.20363244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2717"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 1.446389613s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:34.360465321Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.373361567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2744"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.579734491Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2611"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.753008918Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2411"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.763070988Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3803","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.763101288Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3819"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:34.763110809Z","hostname":"sock","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"wait for service Some(\"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb\") failed: Property not found. retry in 586.307275ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:34.986691463Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 582.518643ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:35.071445124Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.311392284Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3827"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.317611708Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3883"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.449060426Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3945"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.543864926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.566991585Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3973"} {"msg":"wait for service Some(\"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82\") failed: Property not found. retry in 903.036866ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:35.662973789Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.669023829Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4000"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.792268464Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"wait for service Some(\"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de\") failed: Property not found. retry in 1.344485721s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:35.817028025Z","hostname":"sock","pid":1555,"zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.828484672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.860247462Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.860327344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.860727112Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.860735952Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.2.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940646297Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940713129Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940722989Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940730609Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940735279Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940741839Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940749389Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940755729Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940761309Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.9407668Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.94077257Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.94077832Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.94080211Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940807141Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940813591Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940819271Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940824651Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940830121Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940835731Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940841431Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940847071Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940851301Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:35.940856531Z","hostname":"sock","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.180173885Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_bd6e5965-968d-4c3d-aa50-17b3d5747fdb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.180254917Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_33ad5a14-40c4-4b62-8de3-c1143a9861de"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.180261477Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_6c75629f-c9c3-4cd0-8ac6-814984e78c82"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.180278057Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.180544723Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.180817928Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0100256f-8563-45ca-9bc4-1d91a9c62857","remote_addr":"[fd00:1122:3344:101::1]:49431","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9991797,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.18242069Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.258467448Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.334592469Z","hostname":"sock","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.411590006Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.411601366Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.442694733Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.443401037Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.443407157Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.471205079Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.471296961Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:35300"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.471477634Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.471487244Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.492007201Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.492051462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.510889166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.510944607Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.528681729Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.52873727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.528942184Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.529087277Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.540293719Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.573450157Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.574072419Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.57409383Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.617632753Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 205, 184]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.617672834Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"ntp"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.618192535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.618203075Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.169,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.618497541Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.619511651Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.641898705Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:37.660148507Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:38.302928218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:38.30302962Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 57.576188ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:38.436534868Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 119.622093ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:38.508523756Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 271.144416ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:38.659315987Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 509.637736ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:38.940436034Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:39.283592461Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45933"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:39.283850326Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"376f57bd-a764-4881-8255-288226cf17bc","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45933","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":137,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 1.161952438s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:39.460334647Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 1.016057719s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:40.631809316Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 682.153416ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:41.656164436Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8\") failed: Property not found. retry in 1.468580741s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:42.348644393Z","hostname":"sock","pid":1555,"zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.834719712Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.834809944Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.8351346Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.835455496Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"65c95a9f-35ce-408a-aa96-acb14e14b5f1","remote_addr":"[fd00:1122:3344:101::1]:35300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6364128,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.837567109Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.866492652Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.866641325Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40471"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.876270416Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:43.876368538Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"873dd655-addd-4279-a0a7-8a3e958a878c","remote_addr":"[fd00:1122:3344:101::1]:40471","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9645,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:43.878018611Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:44.039957923Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:44.040134147Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:60689"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.048959019Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.049134813Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"571f703f-3393-4152-9a03-be9855571b77","remote_addr":"[fd00:1122:3344:101::1]:60689","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008892,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:48.051989639Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.183949957Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.18407203Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49681"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.193884934Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.194046268Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"4d97878a-b6f1-4d51-8fc3-bd9142a1b65b","remote_addr":"[fd00:1122:3344:101::1]:49681","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9845,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:48.197072318Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.441294422Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.441414235Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:32975"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.449876103Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.450037206Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"0b24dfaf-ea38-4bf5-9d29-196c1b7755bd","remote_addr":"[fd00:1122:3344:101::1]:32975","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8496,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:48.452477614Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.729113792Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:48.729203674Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:40819"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_0516b982-c4cb-4196-b8d1-f13d36b135a8': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:49.737863813Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:49.737966775Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"39a4712e-cec1-4f5d-a472-92962f1c4ba4","remote_addr":"[fd00:1122:3344:101::1]:40819","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":1008678,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:49.742096986Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.055690987Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.05584158Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.064857429Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"1d07f28b-f622-43a3-9849-cd79119e14a1","remote_addr":"[fd00:1122:3344:101::1]:50251","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8958,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.068523612Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:50.068529552Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.339774873Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.339870944Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48671"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.348091818Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"e85254f1-a74e-4906-be58-07e219acb5c6","remote_addr":"[fd00:1122:3344:101::1]:48671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8157,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.351493895Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:50.351501235Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.607072505Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.607166287Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64482"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.615687416Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"22b3eb67-d503-4c5c-bcba-bf6b62bac69c","remote_addr":"[fd00:1122:3344:101::1]:64482","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8379,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.617803228Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:50.617818518Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.902760871Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.902848072Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.911463353Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"624e57b6-14ff-4eff-89bf-a045560ca3d7","remote_addr":"[fd00:1122:3344:101::1]:47820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8492,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:50.912636097Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:50.912643407Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.207179439Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.207298002Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52715"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.216256049Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"d3ae31a4-4d7d-46a0-8e8d-0e0965bc1783","remote_addr":"[fd00:1122:3344:101::1]:52715","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8866,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.219640216Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:51.219648207Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.676737154Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.676820816Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:44160"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.684638251Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"44364170-0627-45c1-a5dd-c7e8af12c4a5","remote_addr":"[fd00:1122:3344:101::1]:44160","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7737,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:51.686544079Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:51.686555309Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:52.252858812Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:52.252958994Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:52.260975503Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"53f79b90-006c-41ad-ab79-d029554dff36","remote_addr":"[fd00:1122:3344:101::1]:33308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7931,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:52.263140116Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:52.263146746Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:53.228135879Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"363.805132ms"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:53.374542613Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:53.374642195Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:53.382691185Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"192a57a1-3090-4466-8f48-0ac0acfa9107","remote_addr":"[fd00:1122:3344:101::1]:47145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7951,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:53.385694284Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:53.385699735Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:54.408138997Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:54.408244579Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:50015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:54.416851739Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"7244b353-b6ff-46d9-a3ec-a2664f9d8b20","remote_addr":"[fd00:1122:3344:101::1]:50015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8437,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:54.420061853Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:54.420070623Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.273593557Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.27373498Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:63814"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.293991022Z","hostname":"sock","pid":1555,"uri":"/timesync","method":"GET","req_id":"e9c80faf-9b38-49ca-aa6a-13e334c91b5d","remote_addr":"[fd00:1122:3344:101::1]:63814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":20178,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1719005874.239101, correction: -0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.297837948Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.297847339Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.326268142Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.326366414Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57987"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.32717227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.32717815Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.346614876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.346673657Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.364621513Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.364670854Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.383499778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.383544088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.469049625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.469099516Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.546846548Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.546896739Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.63264579Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.632716562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.713076136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.713132987Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.797437369Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.79749608Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.797759576Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.797899138Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.883540697Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.883620809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.883635949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.883648989Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.88365968Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.975071883Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.979387479Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.98399311Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.988765105Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.99358684Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:55.997961687Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:56.002762802Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:56.007577058Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:56.012173609Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:56.016294091Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:58.972170672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:58.972327195Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:58.978145201Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:58.978284953Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:59.019550442Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:59.019668774Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:59.080550062Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:59.080699265Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:59.088194064Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:37:59.088319046Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 70.492659ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:59.126521104Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 74.382ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:59.134091414Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 57.108415ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:59.163657261Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 67.471843ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:59.301034405Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 111.004494ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:59.589502498Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 85.120564ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:59.750699206Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 74.555535ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:37:59.89964844Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 106.377753ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.060758266Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 295.762935ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.140257383Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 297.323673ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.40326532Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 53.651114ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.509396736Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 122.759142ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.592654357Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 279.871481ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.640743382Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 428.726711ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.684484579Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 82.303575ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.727059624Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 253.349786ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.765053557Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 300.100452ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.84686834Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 196.822472ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:00.946041368Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.039767474s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:01.062392916Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 530.693888ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:01.125657891Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 902.665245ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:01.348065902Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 720.167613ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:01.38071524Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 225.389355ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:01.419431318Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 768.704527ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:01.665904808Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.094409728s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:01.6861591Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 668.02131ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:02.111499617Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.458850604s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:02.122132128Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.443079895s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:02.262371579Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.047059412s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:02.445511132Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.339044907s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:02.789827993Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 538.642175ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:02.797801601Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.490357758s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:03.345444865Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 563.639769ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:03.501842647Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.388983977s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:03.589607078Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 898.342836ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:03.713522926Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 597.990565ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:04.075332534Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 581.873932ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:04.138257042Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 977.898724ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:04.621257793Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 787.59321ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:04.681358105Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 623.748352ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:04.728704835Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 538.717411ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:04.845707746Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.42500721s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:04.987742493Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.149332265s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:05.360736972Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.243839422s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:05.391752437Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.360745273s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:05.477622721Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 843.02537ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:05.607300013Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 682.166379ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:06.42277741Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 744.936801ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:06.458601831Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 531.526134ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:06.51807304Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 815.333021ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:06.643231483Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.400808732s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:06.847028836Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 726.949156ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:07.057678055Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.424329527s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:07.113067014Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.154469781s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:07.210882223Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.275509656s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:07.466761259Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 886.863378ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:07.79291088Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.189769801s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:08.257343222Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.48750843s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:08.374317013Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 930.701284ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:08.545947587Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.295612709s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:08.688805312Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.358146998s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:08.750975385Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:08.866504427Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"748.470692ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.097142601s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:09.457084402Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 830.527327ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:09.485895003Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:09.558681287Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:09.558865371Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4a2cbd6e-141a-46c8-9f6c-eddd6f0a27f0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36324","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.132602858s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:09.870747918Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.119747395s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:09.992432792Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 624.787598ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:10.11785276Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 931.120645ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:10.325307005Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 796.486831ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:10.562922558Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.321874414s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:10.751432288Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.324933773s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:11.013066018Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 839.795027ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:11.121583641Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 545.406511ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:11.265104887Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 788.748911ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:11.368007739Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 694.66645ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:11.819012666Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 718.437466ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:11.969609033Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 638.432123ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:12.08136936Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 863.367372ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:12.166903797Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 591.726736ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:12.347228733Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.313946282s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:12.522684564Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.428312622s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:12.697786708Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.494448565s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:12.735473355Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.276644931s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:12.94992038Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.05798296s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:13.04019878Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 577.338846ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:13.847139298Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 887.241308ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:14.109010533Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 563.469405ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:14.136630781Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.038143518s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:14.235658515Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.318692239s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:14.244178714Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.123183952s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:14.434508199Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 507.392276ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:14.71080509Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.226824692s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:15.005727441Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.491527046s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:15.227423158Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 769.271375ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:15.283406709Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 1.15982121s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:15.567496264Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 797.510496ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:15.574810219Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 584.122179ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:16.061526725Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 530.578279ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:16.240847751Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.428979638s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:16.381655285Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 625.568213ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:16.654722262Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.260664029s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:16.72873746Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 801.370531ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:16.73681718Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 761.144978ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:16.779603099Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d\") failed: Property not found. retry in 1.483480508s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:17.290489573Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 592.364189ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:17.547084044Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2\") failed: Property not found. retry in 1.260816843s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:17.554654854Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc\") failed: Property not found. retry in 1.128407061s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:17.819360235Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.249531461s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:17.998394697Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9\") failed: Property not found. retry in 652.747636ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:18.149217529Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.223597188s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:19.258801429Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.260293023s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:20.491482232Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 528.722979ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:21.762604087Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.026926814s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:22.311943004Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 998.177981ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:23.349543267Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.320385875s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:24.368419779Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:24.653923583Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.435655619s"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 1.308398959s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:25.699135507Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 562.03608ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:27.018701073Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348\") failed: Property not found. retry in 631.549951ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:27.593386983Z","hostname":"sock","pid":1555,"zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.246155922Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_bd58e908-3308-4fab-8db4-ed67150a5a5d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.246222633Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_c999f854-acb5-47bf-95d4-4879391f2348"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.246231203Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_6ec51f86-b813-4d2c-bee0-1400d33a00d2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.246237423Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_0aa1beae-e845-4690-9fd2-bf35912e37fc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.246243354Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_9a9094ea-c1a1-4c5e-be6a-e79e1be407c9"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.246296305Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.247196593Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.248235403Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"394fae0b-2fbd-433d-beb2-11256193b8da","remote_addr":"[fd00:1122:3344:101::1]:57987","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":32944936,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.279666237Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48015"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::7]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.308821595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3367"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:28.477546432Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3385"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.893403563Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.893506145Z","hostname":"sock","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"e5851eb8-592c-4a59-88a1-8212bd3c38bc","remote_addr":"[fd00:1122:3344:101::1]:48015","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3613779,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.925687673Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.925895598Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:33425"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.926383637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.926399528Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.949138889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.94918721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.969329139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.96936872Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.98853269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:31.988573101Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.006732471Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.006775362Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.024669187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.024705848Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.042106143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.042153134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.060674541Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.060709612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.07926641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.079305841Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.142052266Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.142101907Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.216842729Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.21688514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.288613243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.288665633Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.35906446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.359110761Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.429562099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.429610809Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.507547246Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.507607387Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.577271519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.57732737Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.648789288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.648837409Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.709367859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.70941889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.768946921Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.768993582Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.838839278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.838915799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.897821018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.897867669Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.898111094Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.898307437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993776941Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993831562Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993840013Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993845943Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993852923Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993858773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993869153Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993875253Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993901824Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993907774Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993913344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993919124Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993926114Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993931684Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993937175Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993941075Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993948425Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993955585Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.993961225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:32.994120158Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5f8fcb66-4104-4ff4-a3c0-bd9c093bd934","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized)]","latency_us":1068037,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:32.994383443Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"54.531791ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5f8fcb66-4104-4ff4-a3c0-bd9c093bd934\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5f8fcb66-4104-4ff4-a3c0-bd9c093bd934\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.050503547Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.050886804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.050901454Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.073520603Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.073564754Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.092671313Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.092709604Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.111788302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.111831343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.129163507Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.129204258Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.147575322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.147610023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.165276474Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.165317684Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.183940824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.183979705Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.204326728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.204367919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.224869315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.224920926Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.242725059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.24276687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.261163355Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.261204386Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.279241224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.279276284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.297498086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.297533817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.315193247Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.315236868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.331619043Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.331650933Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.348232862Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.348269283Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.365075757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.365110297Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.38540418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.38543837Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.405402386Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.405439277Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.424942794Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.424977065Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.425205099Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.425388833Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523442748Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523503039Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523511189Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523517169Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.5235228Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.52353021Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.52353596Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.52354173Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.52354911Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523565061Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523571461Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523578631Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523584241Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523589641Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523596741Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523602781Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523612411Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523635192Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.523641882Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.523793185Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4a9ddb30-fd55-4f97-be97-575f9df0ca47","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized)]","latency_us":473107,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:33.52403553Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"77.420509ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4a9ddb30-fd55-4f97-be97-575f9df0ca47\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4a9ddb30-fd55-4f97-be97-575f9df0ca47\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.602202881Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.603071998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.603085588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.625033703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.625086804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.647670952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.647719044Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.669461445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.669510166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.690924241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.690970911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.709719363Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.709769284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.729544057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.729593278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.748917011Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.748968342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.770301595Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.770352616Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.791584117Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.791638288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.812761998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.812808148Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.834262554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.834309625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.852971125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.853022086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.872046443Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.872096475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.890926768Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.890990399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.909639039Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.90968811Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.929735428Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.929774909Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.949821706Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.949865217Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.969863884Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.969904585Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.990052775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:33.990098025Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.009848927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.009916749Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.010125693Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.010253525Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107492334Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107538175Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107544955Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107548625Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107553826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107558026Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107562276Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107566436Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107570706Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107591746Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107596436Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107599816Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107603907Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107607256Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107610487Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107614617Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107618767Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107621967Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107628807Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.107762239Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d0fc0449-0cb1-40da-8418-313594cf3dc2","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized)]","latency_us":505415,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.107987514Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"101.979999ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d0fc0449-0cb1-40da-8418-313594cf3dc2\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d0fc0449-0cb1-40da-8418-313594cf3dc2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.21116781Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.211497727Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.211522767Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.233890821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.233926512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.256735134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.256770985Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.276291012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.276329963Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.296601055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.296636686Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.315547141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.315584652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.335189401Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.335224591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.358030064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.358067824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.379773625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.379809536Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.399848183Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.399892534Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.420063084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.420106095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.440144583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.440181723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.459084908Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.459122419Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.476048685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.476089226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.496113973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.496155504Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.51310554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.513137171Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.531452614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.531490315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.55290656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.55294359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.573152171Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.573184262Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.592615897Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.592654718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.612035742Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.612075223Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.612308898Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.61244029Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706309443Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706357544Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706364344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706369714Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706374104Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706378294Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706383204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706387344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706390674Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706404134Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706408585Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706412875Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706417045Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706421085Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706425175Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706430235Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706435425Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706439625Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706444055Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.706552397Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"63a79cd1-5400-44fe-998c-727fd3773451","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized)]","latency_us":495249,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:34.706759192Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"180.514598ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"63a79cd1-5400-44fe-998c-727fd3773451\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"63a79cd1-5400-44fe-998c-727fd3773451\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.88817303Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.888566598Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.888582459Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.913298919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.91334204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.935822216Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.935858336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.957060957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.957100478Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.97739835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.977432061Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.996326146Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:34.996359086Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.017357613Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.017395014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.038088764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.038122265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.059907167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.059933288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.080878073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.080913884Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.101048203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.101089444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.121592641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.121629142Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.1412169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.141253231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.160042124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.160077014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.17952452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.179568931Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.197723691Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.197755622Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.218905421Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.218947672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.237925648Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.237958539Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.257378334Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.257410155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.277391021Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.277432082Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.295938769Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.29597353Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.296208804Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.296338707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389925173Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389972964Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389978534Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389982965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389987215Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389992395Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389995785Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.389999805Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390003805Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390022585Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390027006Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390030195Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390035226Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390040426Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390044406Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390049456Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390053546Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390058546Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390062926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.390171038Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ef573358-6fa5-42df-96af-158b71fa793a","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized)]","latency_us":501828,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:35.390399943Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"196.106866ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef573358-6fa5-42df-96af-158b71fa793a\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef573358-6fa5-42df-96af-158b71fa793a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.587626625Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.588710967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.588736547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.610833656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.610874667Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.631137949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.631173279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.65135463Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.65139098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.670759524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.670793065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.690502006Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.690540637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.711143696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.711184166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.732083141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.732119982Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.754407304Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.754450655Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.77588954Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.775929051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.796881817Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.796915907Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.818201609Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.81824109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.83938562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.83942577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.860526829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.86056852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.880302091Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.880346092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.900520392Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.900557713Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.920428477Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.920469928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.940477605Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.940514556Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.9613795Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.961420641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.982951118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:35.983002519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.002672289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.00271244Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.002949424Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.003098187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110281743Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110328204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110334604Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110340195Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110344645Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110349945Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110354135Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110358285Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110363495Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110376976Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110380416Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110384556Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110389486Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110393736Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110397916Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110401976Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110406176Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110411006Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110416156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.110531999Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8e14fcad-ae3f-401a-8a17-250412a17dbb","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized)]","latency_us":522735,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.110732312Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"365.885142ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8e14fcad-ae3f-401a-8a17-250412a17dbb\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8e14fcad-ae3f-401a-8a17-250412a17dbb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.478259033Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.47859263Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.47861273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.501513984Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.501553345Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.52348183Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.523518741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.54413154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.544170661Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.564652567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.564690408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.585558282Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.585613133Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.605188981Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.605234552Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.624733189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.62477476Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.644992621Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.645031921Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.665172391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.665212512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.686998414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.687039065Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.707137143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.707174154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.727470867Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.727515818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.749064855Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.749100776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.768424239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.76847565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.788470577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.788509288Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.807127227Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.807169668Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.825580233Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.825616324Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.84561187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.845647301Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.865929373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.865967604Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.886162275Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.886210606Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.8864522Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.886599323Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983524046Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983570707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983576347Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983581577Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983586087Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983591407Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983594788Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983599788Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983604008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983625618Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983630998Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983635159Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983638328Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983642308Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983645559Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983648709Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983651879Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983655849Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.983659029Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:36.983797232Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2871ff30-1574-4657-a70b-1499c32b1d8d","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized)]","latency_us":505391,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:36.984022156Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"184.922039ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2871ff30-1574-4657-a70b-1499c32b1d8d\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2871ff30-1574-4657-a70b-1499c32b1d8d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.17022155Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.170574967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.170589597Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.192329199Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.192372439Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.214067719Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.21410257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.234725579Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.23476Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.253522842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.253556562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.272007078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.272049869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.291853762Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.291890383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.310444521Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.310487602Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.33008318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.330122631Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.350557417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.350590837Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.370355599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.37039167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.390834245Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.390872276Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.410919114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.410954195Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.431456402Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.431487842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.452169502Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.452202123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.472604278Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.472641128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.493918421Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.493958061Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.516205063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.516244824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.535551596Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.535584577Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.556274898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.556326939Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.577967928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.578010899Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.578249043Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.578375096Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674893701Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674943232Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674949642Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674954202Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674959362Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674963632Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674968792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674972092Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674976242Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674981292Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674993603Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.674998923Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675003373Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675008313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675013333Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675017403Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675022363Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675026543Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675031473Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.675137965Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"123e72d0-5f8d-4756-804f-30e6c81ea0a1","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized)]","latency_us":504762,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:37.675341789Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"249.008066ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"123e72d0-5f8d-4756-804f-30e6c81ea0a1\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"123e72d0-5f8d-4756-804f-30e6c81ea0a1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.925565443Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.92591721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.925940741Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.947678792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.947716303Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.967799381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.967836712Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.985564094Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:37.985598964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.004811485Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.004847246Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.023651089Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.02370342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.044406391Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.044447092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.064942388Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.064979629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.084146629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.08418192Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.105298949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.10533361Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.128166012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.128205223Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.14871538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.148748111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.169080544Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.169119255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.188850526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.188885177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.207904464Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.207945505Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.226335679Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.2263724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.245658133Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.245696144Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.264819833Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.264859914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.281632356Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.281669087Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.300065212Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.300112283Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.321557108Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.321594499Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.321842634Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.321981097Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416866399Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.41692255Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416929351Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416934981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416940531Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416946341Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416953281Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416958961Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416964741Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416971621Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.416995592Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417001192Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417006562Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417013332Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417018872Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417024722Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417031242Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417035313Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.417038963Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.417128604Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"45e358d1-e139-47b0-a754-22fda10becc0","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized)]","latency_us":491412,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:38.41743221Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"519.52932ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"45e358d1-e139-47b0-a754-22fda10becc0\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"45e358d1-e139-47b0-a754-22fda10becc0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.937745652Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.938600429Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.938611289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.963878581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.963931412Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.984395438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:38.984440209Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.005764781Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.005829253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.028765998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.028827169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.050281815Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.050337446Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.069884044Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.069929285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.090824099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.09087119Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.111731384Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.111780415Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.133110128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.133154079Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.153421001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.153464351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.174745694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.174798575Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.196087107Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.196132438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.217011632Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.217060373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.23913477Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.239179551Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.258822431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.258867492Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.279184255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.279231766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.300204552Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.300247913Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.319878942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.319919853Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.339734346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.339789927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.360692952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.360744583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.360963347Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.36108653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461029992Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461075053Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461080593Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461086033Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461090374Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461094714Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461100114Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461103384Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461108744Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461112934Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461124644Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461128214Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461132434Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461136674Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461140965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461144265Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461149245Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461153725Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461157905Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.461288348Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d87ff845-959d-44da-9ab3-eae3829e89ea","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized)]","latency_us":523394,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:39.461512052Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"307.094068ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d87ff845-959d-44da-9ab3-eae3829e89ea\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:39 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d87ff845-959d-44da-9ab3-eae3829e89ea\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.589682665Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61736"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.589762806Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f4cb3f97-33df-4d4e-9b42-7844fc5eb7ef","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:61736","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.770053373Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.77042051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.77043799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.793718732Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.793763273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.812917313Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.812953404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.830963801Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.830996952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.848429408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.848476249Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.867676399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.86770983Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.886329519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.88636502Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.905186834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.905221444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.923917985Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.923950786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.943934802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.943981523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.963415898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.963446679Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.986636719Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:39.98667766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.009765928Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.009799099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.032109141Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.032145862Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.054060357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.054095368Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.075446361Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.075482462Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.095467238Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.095500309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.116584617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.116616668Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.137442191Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.137479062Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.15757417Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.157607641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.1772123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.177251451Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.177479955Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.177606818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272716294Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272762835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272769165Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272773625Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272777845Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272782825Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272788425Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272810926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272815586Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272819736Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272823696Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272828136Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272833846Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272838036Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272842916Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272848216Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272853087Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272856297Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.272861107Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.272953169Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4a7db128-3aa3-4fe3-baff-779680c796cc","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized)]","latency_us":502745,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:40.273168253Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"693.433005ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4a7db128-3aa3-4fe3-baff-779680c796cc\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4a7db128-3aa3-4fe3-baff-779680c796cc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.968472506Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.969519877Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.969533887Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.991272688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:40.991318829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.013365037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.013406137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.038201699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.03824183Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.058241977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.058291988Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.080055849Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.08010077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.100479515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.100514555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.121089354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.121124694Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.127850568Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.077173041s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.143967757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.144015628Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.163524915Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.163559926Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.184479881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.184517852Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.203433237Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.203470658Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.221556866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.221587467Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.241011722Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.241048393Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.259849716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.259881256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.282642978Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.282676279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.30442351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.304456831Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.324486398Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.324521319Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.343354912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.343392033Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.364136515Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.364169335Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.383657922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.383811065Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.384222423Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.384413347Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.48189218Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481941341Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481948121Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481952292Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481957502Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481971412Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481976972Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481981252Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481985372Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481989482Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481994642Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.481998893Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.482002003Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.482007003Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.482011253Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.482016043Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.482021063Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.482025473Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.482030653Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:41.482146225Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"94b28a91-fb92-4b03-9dbc-9f072c141199","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized)]","latency_us":513530,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:41.48236032Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"521.220871ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"94b28a91-fb92-4b03-9dbc-9f072c141199\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:40 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"94b28a91-fb92-4b03-9dbc-9f072c141199\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.004871505Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.005207972Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.005217902Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.027249689Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.02729367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.049275416Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.049310037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.06861427Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.06865136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.089254919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.08929221Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.110229315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.110266586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.130208641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.130256653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.151397142Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.151433642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.171097273Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.171144093Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.193207261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.193254162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.213116936Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.213150156Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.23404012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.234077571Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.255589558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.255627309Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.27583511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.275889331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.296377497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.296420608Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.315870864Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.315914255Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.334958002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.334994023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.35399206Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.354026771Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.375070298Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.375108259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.395121396Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.395158457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.413033901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.413075482Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.413346377Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.413514041Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.513893672Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.513954503Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.513963013Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.513968824Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.513976174Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514004394Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514010014Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514015505Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514020815Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514027605Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514032945Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514039495Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514044875Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514050065Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514058035Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514064855Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514070656Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514074276Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514078036Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:42.514170648Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c80a9bfc-f975-4fe9-a1a3-1da107d4beb5","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized)]","latency_us":509168,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:42.514416862Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"638.522127ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c80a9bfc-f975-4fe9-a1a3-1da107d4beb5\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:42 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c80a9bfc-f975-4fe9-a1a3-1da107d4beb5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.153805746Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.154172574Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.154180514Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.176855024Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.176909645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.198307599Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.19835587Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.220001389Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.2200614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.242860723Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.242911134Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.265472941Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.265529693Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.287155511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.287207843Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.307425924Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.307470445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.327960801Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.328005072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.348237343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.348282624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.368160318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.36821007Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.389882569Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.38992602Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.409713553Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.409761254Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.429267851Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.429320342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.449259057Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.449307858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.470106431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.470151052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.491549036Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.491595367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.512871879Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.51291768Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.532954378Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.532997139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.554624388Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.554670958Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.57794225Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.577992901Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.578232626Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.578356078Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681111277Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681158118Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681164308Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681168568Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681173818Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681187958Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681193798Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681198348Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681203408Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681207848Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681213089Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681218029Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681222339Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681226389Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681230489Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681235479Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681239659Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681243679Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681246949Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:43.681389982Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a5cfb007-f6ca-47da-9c9f-f73e42cacae7","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized)]","latency_us":527439,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:43.681610716Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.475820776s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a5cfb007-f6ca-47da-9c9f-f73e42cacae7\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a5cfb007-f6ca-47da-9c9f-f73e42cacae7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.1582843Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.158618966Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.158630676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.180916829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.180960759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.202819973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.202859654Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.222088955Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.222126256Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.242266115Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.242300946Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.263846153Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.263881804Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.286179756Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.286216537Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.306376857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.306414948Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.326771902Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.326811142Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.346925491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.346958542Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.366436529Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.366474189Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.387543407Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.387586588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.408414071Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.408458962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.428700164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.428750994Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.448722331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.448761022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.470831629Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.47086711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.492334216Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.492368257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.512974265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.513010116Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.533289538Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.533331159Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.553211254Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.553247274Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.574118838Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.574157699Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.574417734Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.574573297Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677103581Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677184643Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677193863Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677201793Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677208903Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677214183Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677238634Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677244204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677250774Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677256234Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677261355Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677268655Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677274495Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677279835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677286675Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677292195Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677297495Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677301025Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677306385Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:45.677395067Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a0b297ac-b192-460f-8907-879aa24cd4af","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized)]","latency_us":518969,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:45.677652122Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.576384694s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a0b297ac-b192-460f-8907-879aa24cd4af\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a0b297ac-b192-460f-8907-879aa24cd4af\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.255275558Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.256308318Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.256320638Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.278553549Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.2786126Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.296372063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.296416114Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.315088644Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.315136365Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.334094331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.334144692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.355911404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.355957535Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.376146585Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.376189886Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.396467378Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.396509919Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.418482775Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.418524226Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.44244378Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.442495881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.464372966Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.464419056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.485902762Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.485954054Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.510716645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.510768576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.532970546Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.533023137Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.555903711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.555950402Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.577692733Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.577740144Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.59919219Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.599236711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.621987162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.622039933Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.643970948Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.644031519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.664147289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.66419625Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.684705766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.684763167Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.684999962Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.685122174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785037607Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785080728Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785086188Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785090778Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785094998Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785098298Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785111908Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785117188Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785121528Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785126728Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785131038Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785135889Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785140079Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785144159Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785148209Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785152459Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785157399Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785161529Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785166359Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:47.785249951Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a056a2af-57db-4087-acd1-8a69df8cbee1","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized)]","latency_us":529840,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:47.785470695Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.500974807s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a056a2af-57db-4087-acd1-8a69df8cbee1\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:47 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a056a2af-57db-4087-acd1-8a69df8cbee1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.287279536Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.287708615Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.287715205Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.311705301Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.311745702Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.33231555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.332352461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.353076052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.353106522Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.372463876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.372517207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.391532554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.391564655Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.412084372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.412116743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.431057668Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.431089319Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.450668957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.450704768Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.469510201Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.469540792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.490232672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.490261733Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.511685298Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.511717548Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.531215715Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.531249776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.55313053Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.553174761Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.573039175Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.573071626Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.591042162Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.591076003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.608730263Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.608761414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.627876003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.627908844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.646043713Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.646081584Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.663960289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.663998529Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.684329173Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.684363584Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.684594998Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.684724431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.78500734Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785058221Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785062821Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785066141Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785069401Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785073501Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785099342Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785104022Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785109272Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785114312Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785118622Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785123652Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785127752Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785132703Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785139613Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785146903Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785154413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785160773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785166073Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:49.785231205Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8919c1a1-c78b-42e8-9587-00a4e8ba87f6","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized)]","latency_us":497757,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:49.785441459Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.714234116s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8919c1a1-c78b-42e8-9587-00a4e8ba87f6\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:49 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8919c1a1-c78b-42e8-9587-00a4e8ba87f6\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.500887918Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.501197914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.501205864Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.522632509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.52269818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.54434646Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.544377721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.563692784Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.563725314Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.581719791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.581766942Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.604245588Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.604282589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.627050991Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.627085901Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.648635789Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.64866969Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.669472782Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.669508813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.692364046Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.692402777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.712301692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.712342063Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.732977992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.733027283Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.754595621Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.754636232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.774791652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.774842913Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.793965532Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.794003933Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.814205373Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.814237524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.832914214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.832946565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.854059224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.854093805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.876660032Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.876708053Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.897044687Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.897080787Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.916974332Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.917011443Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.917246358Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:51.91737077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017492156Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017550697Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017557707Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017563637Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017568377Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017573488Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017586448Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017591958Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017595948Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017599978Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017604178Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017608308Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017613569Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017617869Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017623029Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017628049Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017632199Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017637999Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017642479Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:52.017749571Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2724be82-99f6-4af0-a952-86f6da73c59b","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized)]","latency_us":516729,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:52.017957935Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.813672613s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2724be82-99f6-4af0-a952-86f6da73c59b\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2724be82-99f6-4af0-a952-86f6da73c59b\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.833364025Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.833744282Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.833755253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.856248219Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.85630043Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.878228315Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.878271316Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.898096619Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.89813623Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.917159657Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.917212268Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.937715285Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.937760666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.957247792Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.957297043Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.975685818Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.975726899Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.995507601Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:54.995551722Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.016865155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.016911696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.042118186Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.042167067Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.064379357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.064424198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.085993656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.086044937Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.106516863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.106560724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.126417028Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.126455699Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.1466903Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.146735591Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.166957672Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.166997633Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.185831647Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.185875808Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.205244192Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.205290803Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.224419472Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.224459033Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.246625612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.246669323Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.246921448Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.247068751Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341450293Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341502825Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341508025Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341516445Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341521015Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341525985Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341547986Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341553736Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341559306Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341563736Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341569006Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341573466Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341578706Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341584236Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341589486Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341594996Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341600856Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341605337Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341612027Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:55.341682268Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e458dedb-4ab5-4fb2-b14e-4541bb9b24e1","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized)]","latency_us":508189,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:55.341886682Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.27726283s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e458dedb-4ab5-4fb2-b14e-4541bb9b24e1\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e458dedb-4ab5-4fb2-b14e-4541bb9b24e1\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:57.257610354Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"4.527545968s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.620021061Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.620322947Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.620330847Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.644081118Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.644131589Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.664077955Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.664122346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.685085601Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.685131413Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.704608559Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.70465173Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.723423512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.723464583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.743135793Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.743178324Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.762188771Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.762233052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.781419023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.781486554Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.804946709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.805008111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.82462925Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.82466799Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.844773089Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.84483533Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.863105933Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.863143354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.880814864Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.880855445Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.899071626Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.899120998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.917023073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.917082534Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.93552553Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.935577851Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.954478856Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.954521526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.972823829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.97286523Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.991347187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:58.991382768Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:59.009854234Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:59.009892695Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:59.01016612Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:59.010303703Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.10391119Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.103961721Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.103969371Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.103980121Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.103997642Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104009072Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104020632Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104027792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104033392Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104039472Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104051483Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104057173Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104063843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104069133Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104075383Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104081093Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104087504Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104098524Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.104105204Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:38:59.104231656Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6971d4f3-5d86-41ad-ac50-1511ca4ab87a","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized)]","latency_us":484077,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:38:59.10442102Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.303456572s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6971d4f3-5d86-41ad-ac50-1511ca4ab87a\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:38:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6971d4f3-5d86-41ad-ac50-1511ca4ab87a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.296811145Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.297389346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.297402547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.321464444Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.321524555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.342806037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.342845998Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.362899626Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.362954707Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.383552176Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.383608177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.402022392Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.402065553Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.420464418Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.420513869Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.438256461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.438295452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.457228207Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.457281388Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.475256895Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.475298776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.493283103Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.493321243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.513985383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.514031324Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.532370878Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.532429299Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.551174311Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.551227442Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.569373002Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.569407873Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.587072713Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.587113324Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.605218763Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.605263834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.623401524Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.623435734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.643519743Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.643561704Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.661956289Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.661993669Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.680384734Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.680420585Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.68066026Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.680793722Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779184344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779249685Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779266246Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779274596Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779297726Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779331477Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779341387Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779351037Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779361047Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779375238Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779386918Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779396648Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779409999Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779418339Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779423879Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779438239Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779446719Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.7794639Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.77947034Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:01.779525701Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4f0b2f10-dffb-450e-bd0f-d285de807755","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized)]","latency_us":482482,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:01.779763855Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.001349973s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4f0b2f10-dffb-450e-bd0f-d285de807755\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4f0b2f10-dffb-450e-bd0f-d285de807755\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.774689555Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.775194405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.775202676Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.797609491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.799975049Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.818478806Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.818531958Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.837719889Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.83777766Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.857261922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.857312259Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.875942129Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.87598971Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.894936557Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.894992408Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.912835773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.912890475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.931385352Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.931451344Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.949627675Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.949681136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.968519911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.968590123Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.987017179Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:04.98707164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.005655Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.005701201Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.028870202Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.028935003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.04741022Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.047462851Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.066122912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.066174424Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.083980438Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.084030169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.102423624Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.102477965Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.120294709Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.12034473Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.138740407Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.138784357Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.157395167Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.157447288Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.157728034Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.157889977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251712724Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251763555Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251770865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251775495Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251783915Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251796835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251801365Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251805985Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251810545Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251818646Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251823596Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251828956Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251833676Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251838976Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251842816Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251847986Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251853146Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251857486Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.251862867Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:05.251981499Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9591452e-94d7-4a2e-b4b7-90234d9ccf57","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized)]","latency_us":478401,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:05.252193872Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.472152315s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9591452e-94d7-4a2e-b4b7-90234d9ccf57\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:05 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9591452e-94d7-4a2e-b4b7-90234d9ccf57\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.71844326Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.718994511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.719007821Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.741366326Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.741405922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.759998666Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.760040567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.780441753Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.780480174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.798509642Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.798549283Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.817202914Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.817254685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.836813764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.836863685Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.854849343Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.854887314Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.872866992Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.872914702Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.891781778Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.891826358Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.90997527Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.910018491Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.928213612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.928264533Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.949011056Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.949051337Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.968274329Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.96832526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.986507682Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:07.986546673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.004729214Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.004770875Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.025240492Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.025287453Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.043965695Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.044007956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.062007154Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.062045835Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.079831038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.079866158Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.098873776Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.098927677Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.099181523Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.099316675Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191123842Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191181888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.19119101Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191199345Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.19120601Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191237704Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191253614Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191260494Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191275865Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191283175Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191291665Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191300405Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191306985Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191315765Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191322846Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191331376Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191338076Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191346446Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191382887Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:08.191504709Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"227b936b-c2cb-4d7b-a6c6-85af565b2e38","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized)]","latency_us":474145,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:08.191706523Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.608933287s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"227b936b-c2cb-4d7b-a6c6-85af565b2e38\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"227b936b-c2cb-4d7b-a6c6-85af565b2e38\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:09.490409575Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48610"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:09.490492287Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"fd926ab3-ad59-4ca1-8bec-bc8805f8833e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:48610","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.783610963Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.784147923Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.784161493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.806853265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.806905496Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.825049617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.825108088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.846228558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.84628732Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.864867529Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.86492853Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.88303295Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.883078721Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.900387076Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.900451977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.920103098Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.920155829Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.938926322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.938976973Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.959894109Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.96194135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.979174203Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.979236184Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.998675662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:14.998727662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.01863641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.018700529Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.03737045Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.037437512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.057226135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.057280866Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.076937317Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.076995068Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.095033117Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.095079198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.112470784Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.112527075Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.129977702Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.130031853Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.148551171Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.148603342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.166446037Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.166513969Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.166835195Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.167034959Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259450757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259515468Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259533129Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259542019Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259550189Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259557929Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259565869Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.25957384Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.25957989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.25958641Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.25959319Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.25959968Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.25960598Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.25961257Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259620261Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259628341Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259636121Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259659331Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.259668792Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:15.259809174Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"54185345-2720-4b38-8ed5-ffee3a3e1383","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized)]","latency_us":477281,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:15.26007174Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"9.168325066s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"54185345-2720-4b38-8ed5-ffee3a3e1383\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"54185345-2720-4b38-8ed5-ffee3a3e1383\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:16.725792814Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"8.944880162s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.40491378Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.405499631Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.405520952Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.428589791Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.428648612Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.450039957Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.450112959Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.470392562Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.470441403Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.489176106Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.489237717Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.508515911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.508580182Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.526862616Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.526913877Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.544893894Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.544945215Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.56225396Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.562327181Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.580804578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.58086164Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.599544627Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.599608653Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.618840125Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.618898876Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.637407231Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.637467626Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.656040365Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.656089276Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.674529363Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.674589834Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.692194724Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.692273935Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.710668712Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.712978738Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.729245911Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.729310103Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.747317431Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.747371232Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.765727967Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.765785128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.783650153Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.783705265Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.78397515Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.784134643Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878158524Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878211345Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878218545Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878223215Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878233535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878239175Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878243545Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878253145Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878257785Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878263226Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878273666Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878278116Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878284206Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878288426Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878292826Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878305876Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878311616Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878317687Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878322397Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:24.878428129Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7ed224d9-31b6-49a3-a5aa-741b46877794","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized)]","latency_us":474594,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:24.878644833Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"8.806764964s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7ed224d9-31b6-49a3-a5aa-741b46877794\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7ed224d9-31b6-49a3-a5aa-741b46877794\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.661802205Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.662137396Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.662147567Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.68595949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.686030692Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.707217583Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.709802505Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.730256802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.730319673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.752838881Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.752898772Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.77541174Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.775470611Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.796489399Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.79655342Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.816273502Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.816331103Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.836981984Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.837042155Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.857948511Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.858011083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.878448269Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.8785128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.897998598Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.898059649Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.91670497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.916763471Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.9378182Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.937874761Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.960524862Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.960588263Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.985150532Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:33.985221253Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.00871461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.008787641Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.031582925Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.031642466Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.054625673Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.054682204Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.077120001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.077181072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.099360363Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.099424384Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.099670229Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.099830312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.199965114Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200030635Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200036926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200041606Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200046136Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200050856Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200055556Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200060966Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200064756Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200070056Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200077956Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200083676Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200088376Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200092647Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200096887Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200102917Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200125337Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200131027Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200135337Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:34.200225949Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e35f8e3b-45a7-4cc0-8edd-76d7ad55dcb4","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized)]","latency_us":539745,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:34.200430763Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"9.47666852s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e35f8e3b-45a7-4cc0-8edd-76d7ad55dcb4\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e35f8e3b-45a7-4cc0-8edd-76d7ad55dcb4\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:39.442711728Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:39.44281229Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d005ca1c-6e26-4dd2-bf41-bf482573d907","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:41208","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":72,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:40.641064625Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"8.856853868s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.652910324Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.654070977Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.654080617Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.676148286Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.676186497Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.696845578Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.696886949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.71605295Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.716087531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.734007547Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.734051088Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.751441894Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.751501545Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.769501243Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.769547404Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.788137124Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.788172434Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.805739014Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.805778265Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.824287423Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.824330964Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.843487475Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.843535126Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.861320689Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.861372381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.880158194Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.880197415Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.898674563Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.898730334Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.916424956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.916481637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.93525077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.935298331Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.953659966Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.953694927Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.972209135Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.972251916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.990780144Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:43.990818885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:44.008894484Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:44.008946405Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:44.027726509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:44.027799891Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:44.028073476Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:44.02825956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121111007Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121175188Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121194818Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121205049Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121211809Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121217449Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121223939Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121230219Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121241369Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121247989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121272299Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121279734Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121287084Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121293327Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121299393Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121307277Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121312491Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121324811Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121330271Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:44.121389362Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"322036d2-017c-42f0-90b5-cc69fb82b14e","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized)]","latency_us":469577,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:44.121598946Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"7.471388365s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"322036d2-017c-42f0-90b5-cc69fb82b14e\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"322036d2-017c-42f0-90b5-cc69fb82b14e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.57335704Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.573906261Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.573919751Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.59799904Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.598048581Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.617241433Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.617294764Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.636366414Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.636410975Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.655111956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.655149217Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.672643745Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.672710929Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.690662759Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.690712365Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.708872716Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.708908726Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.726206461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.726259962Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.744483477Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.744525885Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.763288798Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.76334602Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.782856437Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.782893808Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.800838215Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.800879296Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.818747662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.818793922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.837073656Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.837120637Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.855079354Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.855120795Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.874067492Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.874108753Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.894360166Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.894398916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.911945025Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.911987916Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.930611667Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.930650857Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.948004113Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.948057844Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.948297408Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:51.948477922Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.03942092Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039493852Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039503842Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039520913Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039527993Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039536403Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039543093Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039556463Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039568993Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039576954Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039589344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039620135Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039635645Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039644355Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039651225Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039660965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039666405Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039671376Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.039678118Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:39:52.039764247Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"da0d5207-e4ae-4664-a82e-7659f94a05e4","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized)]","latency_us":467442,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:39:52.040020912Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"11.596348581s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"da0d5207-e4ae-4664-a82e-7659f94a05e4\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:39:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"da0d5207-e4ae-4664-a82e-7659f94a05e4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.60585752Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.606247858Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.606258558Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.628914728Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.628962649Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.649531099Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.649569479Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.668971001Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.669008846Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.687102136Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.687141287Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.705592644Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.705646555Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.723753975Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.723802976Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.741294054Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.741327345Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.758667969Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.75870576Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.777531395Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.777574076Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.796033683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.796093004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.814324367Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.814362777Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.83259346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.832638241Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.852543077Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.852575997Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.871929893Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.871963493Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.890809898Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.890843859Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.909313163Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.90934712Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.927878735Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.927929346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.946425084Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.946486745Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.964768139Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.964807619Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.98292257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.982969981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.983300187Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:03.9834631Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075317078Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075370408Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075377819Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075384879Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075396159Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075402199Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075407799Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.0754145Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075420989Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.07542763Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.07543532Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.07545093Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075469101Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075475221Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075480621Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075484921Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075489041Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075498761Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075507271Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:04.075558642Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b19a1c97-6acf-42c7-bd74-572df69cd2aa","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized)]","latency_us":470809,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.075771247Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"15.845874026s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b19a1c97-6acf-42c7-bd74-572df69cd2aa\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:40:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b19a1c97-6acf-42c7-bd74-572df69cd2aa\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:04.467764104Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"35.786617309s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:09.392492182Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57825"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:09.393379102Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a3b0d1e8-136d-4f7a-bd0e-ddfa637c8dc0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:57825","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":810,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.88048488Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.880766052Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.880772877Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.901580949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.9016257Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.921797311Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.921854683Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.940938662Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.940978033Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.959031372Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.959080303Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.977248614Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.977298066Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.995436004Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:19.995479177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.014129648Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.014180549Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.032692917Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.032740208Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.0509072Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.05094868Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.070423948Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.070459359Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.089154271Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.089190128Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.106849503Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.106883813Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.125007224Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.125062805Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.143115064Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.143150034Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.160657143Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.160709824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.179395526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.179447336Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.198036796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.198071457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.216502844Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.216562095Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.234336568Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.23438802Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.25201293Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.252065801Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.252430858Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.252623292Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343681583Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343733644Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343748605Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343753535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343757835Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343778305Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343783185Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343787676Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343791246Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343796566Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343806196Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343811626Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343815806Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343821236Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343825516Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343830197Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343839757Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343845277Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.343849797Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:20.343952699Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c914dc52-3b57-493f-9792-bcaf5e911dbc","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized)]","latency_us":464607,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:20.344164803Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"15.333558524s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c914dc52-3b57-493f-9792-bcaf5e911dbc\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:40:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c914dc52-3b57-493f-9792-bcaf5e911dbc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.637194557Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.638019843Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.638029023Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.663171184Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.663229035Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.686409956Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.686455187Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.706565907Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.706615058Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.727666727Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.727717348Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.747323517Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.747362478Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.767086531Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.767130111Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.78663949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.786682051Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.807490305Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.807529645Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.82739202Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.827432751Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.847083461Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.847131122Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.868088749Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.86813069Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.886635018Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.886674679Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.90585073Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.905905842Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.926845038Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.926887409Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.947505949Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.94754322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.967818083Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.967857814Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.987054696Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:35.987093757Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.007346519Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.00740005Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.029421218Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.029462199Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.049137381Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.049179892Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.049464217Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.04962718Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144576609Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144647911Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144657571Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144670841Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144677901Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144685001Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144703841Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144713602Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144728042Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144736482Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144744692Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144752262Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144760023Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144767933Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144776283Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144789413Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144797343Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144802594Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.144813734Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:36.144894555Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f5f8a949-046e-45cc-8d6f-98954f318c82","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\", TimeNotSynchronized), (\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\", TimeNotSynchronized), (\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\", TimeNotSynchronized), (\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\", TimeNotSynchronized), (\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\", TimeNotSynchronized), (\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\", TimeNotSynchronized), (\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\", TimeNotSynchronized), (\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\", TimeNotSynchronized), (\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\", TimeNotSynchronized), (\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\", TimeNotSynchronized), (\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\", TimeNotSynchronized), (\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\", TimeNotSynchronized), (\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\", TimeNotSynchronized), (\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\", TimeNotSynchronized), (\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\", TimeNotSynchronized), (\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\", TimeNotSynchronized), (\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\", TimeNotSynchronized), (\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\", TimeNotSynchronized), (\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\", TimeNotSynchronized)]","latency_us":508953,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:36.145059209Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"21.660533436s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f5f8a949-046e-45cc-8d6f-98954f318c82\", \"content-length\": \"124\", \"date\": \"Fri, 21 Jun 2024 21:40:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f5f8a949-046e-45cc-8d6f-98954f318c82\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:39.34304687Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43859"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:39.343617321Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8258dfdc-3dfd-4fd3-8e99-3a95a0ae3276","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:43859","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":539,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:40:55.149862612Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"37.150103644s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 8d5eb02b-9914-404f-88f5-5b0790974f59, kind: Service { id: 0516b982-c4cb-4196-b8d1-f13d36b135a8 }, name: Name(\"ntp-0516b982-c4cb-4196-b8d1-f13d36b135a8\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 184])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.2.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: bd58e908-3308-4fab-8db4-ed67150a5a5d, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: c999f854-acb5-47bf-95d4-4879391f2348, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: 8118d247-c8a4-47b6-be18-05eb663745b6, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.2.164:53, nic: NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.2.165:53, nic: NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: f3a19a9d-3117-4bf5-aa39-824f48040fae, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.2.166, nic: NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 62aaea40-4286-44d3-b143-827d830cadd5, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.2.167, nic: NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.2.168, nic: NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 6021b400-4bb3-4adf-ac63-5c76d8b2bda6, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 10b348d3-f952-455c-858e-798e7e59ab91, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: ce51ce61-e883-4d12-b122-913a7a0e4a3b, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ab7d0f8d-26db-4202-89ba-241139a6439a, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 8c6fbfe9-c167-4d5c-bf93-ee11dc465b15, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External } } } }, OmicronZoneConfig { id: b88f007b-85df-4a7a-bde1-c68774a9ee57, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External } } } }, OmicronZoneConfig { id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External } } } }, OmicronZoneConfig { id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External } } } }, OmicronZoneConfig { id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External } } } }, OmicronZoneConfig { id: d84da9e5-ea99-4f13-a054-b6abd174b00d, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External } } } }, OmicronZoneConfig { id: bd658f3d-7b61-4998-b6a3-589ca0776868, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External } } } }, OmicronZoneConfig { id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8e952779-16e7-4682-a009-9ce5d6fa3280, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.748560634Z","hostname":"sock","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.74890509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.748913561Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd6e5965-968d-4c3d-aa50-17b3d5747fdb, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.770706674Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.770757325Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6c75629f-c9c3-4cd0-8ac6-814984e78c82, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.787228302Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.787273863Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 33ad5a14-40c4-4b62-8de3-c1143a9861de, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.804810122Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.804849693Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd58e908-3308-4fab-8db4-ed67150a5a5d, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.822871351Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.822908312Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 6ec51f86-b813-4d2c-bee0-1400d33a00d2, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.843595883Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.843633094Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a9094ea-c1a1-4c5e-be6a-e79e1be407c9, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.863638012Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.863688383Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c999f854-acb5-47bf-95d4-4879391f2348, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.884387435Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.884438266Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 0aa1beae-e845-4690-9fd2-bf35912e37fc, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.907533055Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.907582946Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8118d247-c8a4-47b6-be18-05eb663745b6, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.929336279Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.92939652Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.953271565Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.953318786Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 10b348d3-f952-455c-858e-798e7e59ab91, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.9761564Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:57.976204661Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9b262d50-7378-4dec-9a9e-7774b96af0c2, dataset_name: DatasetName { pool_name: ZpoolName { id: 5b452253-5f6d-4617-91b5-3f263edef77d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.000924773Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.000975024Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b88f007b-85df-4a7a-bde1-c68774a9ee57, dataset_name: DatasetName { pool_name: ZpoolName { id: 41b82b1f-5def-467c-bc86-5c87195bd788 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.024792568Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.024836409Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8, dataset_name: DatasetName { pool_name: ZpoolName { id: 44b9c17b-6e37-45fe-b9da-6182ccbe59a4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.047822976Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.047868027Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: a2a67eb9-3e4b-4f09-9ca5-64c505877ca6, dataset_name: DatasetName { pool_name: ZpoolName { id: e6f1c1a1-ab63-45ad-a43f-153910e0494a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.06913797Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.069225512Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 54f5f1b6-b399-4163-a9e3-0b7868c8be76, dataset_name: DatasetName { pool_name: ZpoolName { id: 8ce4d8fb-4017-46e3-ac55-3c40125a531b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.090654058Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.090726169Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: d84da9e5-ea99-4f13-a054-b6abd174b00d, dataset_name: DatasetName { pool_name: ZpoolName { id: a0c5f5f0-1cc5-45cc-9434-17b4abf3b972 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.111308198Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.11136509Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: bd658f3d-7b61-4998-b6a3-589ca0776868, dataset_name: DatasetName { pool_name: ZpoolName { id: 85a7fd4f-88d4-408c-a832-0d457152c72f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.130598092Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.130666152Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 1796d25b-1b93-4fd3-bf11-a40ad30fc679, dataset_name: DatasetName { pool_name: ZpoolName { id: 6b2e5449-891f-4dfd-8e2d-78189eabe443 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.151767003Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.151821944Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 8e952779-16e7-4682-a009-9ce5d6fa3280, dataset_name: DatasetName { pool_name: ZpoolName { id: 3ae04f9e-dd4d-4980-8e86-31c81824c907 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.17021316Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.170267221Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.170650959Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.170800322Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.297314518Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.297629985Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.297744797Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.297787028Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.297814559Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.322112132Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.322319796Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.322360237Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.322369737Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.334161582Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.499248855Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.547909483Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.548152688Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.548180248Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.548209179Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.550859882Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.550970464Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.564093165Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.564138286Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.564304539Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.570070464Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.575146045Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.580625684Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.586025421Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.591318467Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.597326926Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.603114771Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.740690668Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.745125386Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.750319609Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.755950025Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.761736567Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.766315828Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.771834217Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.777966979Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.784203123Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.791047939Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.797457197Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.804072688Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.811100568Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.817194929Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.825829291Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.832932443Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.839821229Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.847293008Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.853770437Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.859427139Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.931274648Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:58.994959925Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.075076765Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.128687488Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.194106906Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.19628222Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.19630727Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.375802331Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 255, 165]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.375889283Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.380766819Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.38080128Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.49573776Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 159, 118]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.495813962Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.500153517Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:40:59.500183548Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.110355122Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 193, 73]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.110459384Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.110486984Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.110502575Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.110527905Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.110541516Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.14204973Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.142087541Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.588214611Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 153, 223]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.588294743Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.592022947Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.592041907Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.74390318Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 241, 8]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.743983291Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"external_dns"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.168,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.744794367Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.167,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.754173554Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.75853824Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:00.758564711Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.682229343Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.682341616Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"Configuring new Omicron zone: oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.683150832Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.733699924Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.733785026Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"no nat entry found for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.734649763Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:102"} {"msg":"creating nat entry for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.734669164Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:109"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.166,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.734714435Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.759865394Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.760327093Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.165,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.785330179Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"creation of nat entry successful for: V4(\n Ipv4Network {\n addr: 10.151.2.164,\n prefix: 32,\n },\n)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.791105473Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:137"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.793827457Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:01.793847177Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Installing Omicron zone: oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:02.700713487Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:02.716227535Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.156216226Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Profile for oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.177519888Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.180380585Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.184353084Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.188251041Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Configuring new Omicron zone: oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.189451995Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.203966433Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.224829957Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.257940824Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.267551984Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.292993319Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Installing Omicron zone: oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.301082199Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.309227331Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.321066746Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.456688226Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53788"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.462952311Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"77db912a-8b08-409c-a2a6-09293d72c9dc","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:53788","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":4751,"response_code":"200"} {"msg":"Profile for oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.657532761Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.658256955Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.6685981Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:09.668963427Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 53.271583ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.818041434Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 47.340087ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.832567252Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 30.154829ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.849262013Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 62.912876ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.871910193Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 121.72442ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.891345228Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 146.90107ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.921261642Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 94.997065ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.957802597Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 59.498054ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:09.9902402Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 243.659852ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.050802642Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 244.853178ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.082877378Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 290.439848ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.110250771Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 120.101204ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.134093344Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 57.727264ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.153486199Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.154854816Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 71.357254ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.174680459Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.174975405Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 93.911853ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.237914254Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 117.058171ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.506712736Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 558.633695ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.655298283Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 373.2347ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.692349568Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 461.137209ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.829814685Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 270.456646ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.849679319Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 325.617954ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.868293008Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 179.94806ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:10.897815824Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.901381555Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.90920231Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.909305202Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.925577384Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.935350949Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:10.93541766Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 40.709753ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.08162606Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 573.843402ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.10229063Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 394.327605ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.135274275Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 423.487376ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.161827421Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 110.146839ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.244591553Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 499.167627ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.261244253Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 867.68131ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.276067237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 1.021419719s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.307399799Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 43.918409ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.36038941Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 199.072404ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.380307495Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 56.398999ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.396927235Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 56.064936ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.412907282Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 148.440498ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.429394499Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 105.000403ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.634805184Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 912.622419ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.653952794Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 362.642892ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.687438768Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 826.578952ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.717705218Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 141.069031ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.747436968Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 296.032196ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.774561686Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 1.028119559s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.805649022Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 796.117912ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.840365331Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 231.061552ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.865379518Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 216.607366ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:11.914546393Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 1.184793821s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.063440007Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 238.210785ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.08884976Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 512.232113ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.112289546Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 365.798054ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.145609886Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 577.620461ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.159067273Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 559.130954ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.343681686Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 1.131781645s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.357587272Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 1.113943543s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.532700016Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 1.04821797s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.568602858Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 957.805549ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.588584614Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 707.910587ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.64129228Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 800.270654ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.657391439Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 1.477863575s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.753546477Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 1.184849248s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.851176923Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:12.852081431Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:12.855038829Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:12.855108561Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:12.864372535Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 1.404116877s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:12.919825875Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 74.235545ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.1469639Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 29.214011ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.170945856Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 127.276593ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.215294436Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 131.410748ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.236936635Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 1.02241259s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.306416944Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 253.051103ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.374733539Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 244.660547ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.407329025Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 668.023224ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.450630884Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 763.397433ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.478083819Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 868.344843ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.506939611Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 811.527275ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.567041074Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 1.349155633s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.637497281Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 576.135666ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.687144416Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 465.402784ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.752806859Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 1.340038222s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:13.778654401Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 621.512167ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.051541155Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 1.241112548s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.13601987Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 449.642759ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.230956004Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 907.842084ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.245026853Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 1.08376848s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.255859838Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 421.349806ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.272681542Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 1.212246616s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.33961916Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 1.288380001s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.354366882Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 680.965092ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.387163123Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 1.485705115s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.39658255Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:14.666955183Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:14.667038255Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 967.686465ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.683501231Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:14.683637344Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:14.68442849Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 922.897663ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.693571461Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:14.693725934Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:14.698267274Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 1.300746576s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.713743801Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 64.58339ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:14.86594135Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 31.371889ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.012255422Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 1.144746669s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.100750128Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 97.19211ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.119919208Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 1.022355724s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.139880554Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 54.488383ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.159099715Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 58.795393ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.184075481Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 520.531589ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.208424724Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 772.661996ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.239263255Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 291.687011ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.265135919Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 73.576904ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.285036654Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 253.281418ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.305741074Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 1.345561751s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.362592212Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 273.262336ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.384387385Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 660.810061ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.399432923Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 1.042109378s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.576533366Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 325.199391ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.597304668Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 549.781414ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.608670194Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 1.456240882s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.630436286Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 981.616391ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.657755828Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 713.089576ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.67101722Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 376.135766ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.683704232Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 684.349813ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.742188883Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 954.3376ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.895651207Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 973.951263ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:15.93515809Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 1.105932108s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.025295528Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 1.178098786s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.036344907Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 510.449407ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.07378669Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 807.465045ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.084789928Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:16.145749308Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:16.147670246Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:16.157854778Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 1.119557429s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.175730663Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:16.175854895Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 1.287879248s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.200568525Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:16.200932432Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:16.202048095Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 790.598254ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.25874823Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 831.459462ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.402323388Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 38.169432ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.426703461Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 1.310703478s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.446621046Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 35.696899ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.472147583Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 83.293256ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.49369832Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 25.677731ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.517781488Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 144.584045ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.538180503Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 141.892458ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.655648113Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 240.534065ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.67566919Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 534.827392ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.694154237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 580.359814ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.706340098Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 891.342252ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.718458169Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 269.665022ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.728297414Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 1.08962326s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.73867208Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 196.675653ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.818121515Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 572.116441ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.869912103Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 503.508823ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.917281262Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 1.015494313s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.935715608Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 429.985023ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:16.957843817Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 269.14761ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.015351698Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 412.187113ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.029911117Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 550.982229ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.065259658Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 1.0488689s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.108419824Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 747.321792ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.154527959Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 1.443398164s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.232618408Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 602.491644ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.254467751Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 916.065497ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.272242184Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 456.88363ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.306264899Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 704.949845ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.322116073Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 963.875266ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.340195862Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 842.890573ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.407456476Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 1.425666676s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.441320038Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 427.039546ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.463010198Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 1.150435091s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.481124968Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:17.485885212Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:17.486205399Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 1.249932277s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.510193564Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 1.363285496s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.631120413Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 673.75777ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.647523199Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 34.364584ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.764327866Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 1.299512537s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.781633439Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 722.158652ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.795098286Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 74.870847ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.811524512Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 563.964786ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.851989824Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 1.335589435s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.925084244Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 1.495439457s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.948470228Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 237.91536ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.960621539Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 1.357924075s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.975392772Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 1.156886647s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:17.988756417Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 741.764049ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.046137486Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 1.055919374s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.174551443Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 1.150079414s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.214615738Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 410.496955ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.230060704Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 1.421135676s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.266956476Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 1.380046221s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.316786735Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 1.493488184s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.335614478Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 671.369726ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.431137923Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 1.172504561s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.53177826Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6\") failed: Property not found. retry in 1.074372358s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.644078008Z","hostname":"sock","pid":1555,"zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 842.148857ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.657149507Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 525.501069ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.692222312Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280\") failed: Property not found. retry in 898.049365ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.775486254Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 1.181289061s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.803861917Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 691.153783ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:18.881216391Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 1.102135176s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.008652149Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 809.722929ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.096558213Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868\") failed: Property not found. retry in 799.048611ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.118672932Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 662.352677ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.157732527Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 1.464991597s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.23298618Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 990.803136ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.2455873Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 900.596369ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.274403171Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 1.153218635s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.352791406Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76\") failed: Property not found. retry in 1.258596341s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.379738941Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57\") failed: Property not found. retry in 1.247528805s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.458387241Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 535.570864ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.515426862Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 1.362263201s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.590497142Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 1.107454544s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.718274086Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 1.207407931s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.733437887Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 839.633493ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.75171474Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 516.353806ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.840242265Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 820.114981ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.856194192Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 534.444339ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:19.928491996Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 786.717832ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.005747399Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 564.736116ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.074466312Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 923.097976ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.125728929Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 1.435595738s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.187946683Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 985.803084ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.250554605Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 1.21500355s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.371608277Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 714.096982ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.480637479Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 973.100289ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.524396208Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 1.342270093s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.607016267Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 734.210254ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.67273023Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 1.389120715s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.688942912Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 776.441563ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.709971089Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 613.886481ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.811969442Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 1.287474554s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.841545788Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 1.12197908s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.960433217Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 1.075488832s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:20.976401324Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 569.690422ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.066507871Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 1.28767986s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.21161963Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 1.247408527s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.250802177Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 684.033063ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.41984594Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 781.535813ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.43849591Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 762.174405ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.501285846Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 553.603773ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.517364415Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 826.782992ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.605019114Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 1.339928927s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.638207262Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 946.202977ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.65372237Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 749.403091ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:21.962563396Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 664.887153ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.067017649Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 1.247195101s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.084272411Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8\") failed: Property not found. retry in 1.469634362s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.098060754Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 1.223657865s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.111216085Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 1.467569998s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.123208763Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 1.043672134s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.1411875Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 1.342281782s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.234193415Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 530.281198ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.276547755Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 685.735052ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.443485347Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf\") failed: Property not found. retry in 1.472757559s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.512090428Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 660.441092ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.522531335Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15\") failed: Property not found. retry in 1.089628445s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.612772815Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 722.353804ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.72646372Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d\") failed: Property not found. retry in 970.476737ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.743907266Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6\") failed: Property not found. retry in 1.270951509s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.820880623Z","hostname":"sock","pid":1555,"zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6\") failed: Property not found. retry in 975.177464ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:22.991244242Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 793.9611ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.141537734Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 1.318240647s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.198492444Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679\") failed: Property not found. retry in 874.236837ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.209511512Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 1.253852016s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.343659973Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae\") failed: Property not found. retry in 683.992357ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.353600451Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 634.122982ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.460669825Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5\") failed: Property not found. retry in 938.082654ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.597692003Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 915.230829ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.608315573Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07\") failed: Property not found. retry in 827.073032ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:23.952191015Z","hostname":"sock","pid":1555,"zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b\") failed: Property not found. retry in 1.350824988s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:24.122689837Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a\") failed: Property not found. retry in 860.422657ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:24.533624069Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 1.246424126s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:24.551701658Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2\") failed: Property not found. retry in 1.156522603s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:24.609117247Z","hostname":"sock","pid":1555,"zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 731.348755ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:25.813807514Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91\") failed: Property not found. retry in 768.273189ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:26.563564417Z","hostname":"sock","pid":1555,"zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.36322203Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_b88f007b-85df-4a7a-bde1-c68774a9ee57"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363304961Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_54f5f1b6-b399-4163-a9e3-0b7868c8be76"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363312192Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_8e952779-16e7-4682-a009-9ce5d6fa3280"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363317662Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_oximeter_6021b400-4bb3-4adf-ac63-5c76d8b2bda6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363329472Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_3bdb0a0f-906b-4629-8c23-bd2bba1cce07"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363335022Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_clickhouse_10b348d3-f952-455c-858e-798e7e59ab91"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363340562Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_ab7d0f8d-26db-4202-89ba-241139a6439a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363345432Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_a2a67eb9-3e4b-4f09-9ca5-64c505877ca6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363355462Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_62aaea40-4286-44d3-b143-827d830cadd5"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363360883Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_bd658f3d-7b61-4998-b6a3-589ca0776868"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363365853Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_external_dns_1acd5e4e-4ad7-4431-b537-a28da2f217bf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363370843Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_a7a0de82-3bf3-4ebb-89a5-e8dc8d06cbe8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363375553Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_d84da9e5-ea99-4f13-a054-b6abd174b00d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363380243Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_1796d25b-1b93-4fd3-bf11-a40ad30fc679"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363386113Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_external_dns_8118d247-c8a4-47b6-be18-05eb663745b6"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363398073Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_8c6fbfe9-c167-4d5c-bf93-ee11dc465b15"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363403263Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_f3a19a9d-3117-4bf5-aa39-824f48040fae"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363408343Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_ce51ce61-e883-4d12-b122-913a7a0e4a3b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.363413284Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_9b262d50-7378-4dec-9a9e-7774b96af0c2"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.364361782Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.368530055Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.37482127Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2a1ca099-b3db-415a-8636-77d56da69b37","remote_addr":"[fd00:1122:3344:101::1]:33425","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":29629653,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.406634761Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.414117489Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.414346464Z","hostname":"sock","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.416729691Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.419322963Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.2.174,\n infra_ip_last: 10.151.2.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.2.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.2.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.465613111Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.473532628Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.624202637Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.703564151Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:27.895819645Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:28.08312442Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:28.241548326Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:28.483141013Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:28.671702452Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:28.857988686Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:29.471822659Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:30.011452186Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:30.447049788Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:31.507015028Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:32.026811225Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:41:32.333435106Z","hostname":"sock","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:417","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:417","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"158.655787672s"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:32.976007225Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/884d2112-d3cd-48eb-9cc7-ecb0a29f78a9/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:34.45913445Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.297173304Z","hostname":"sock","pid":1555,"component":"NexusNotifierTask","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.434805474Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48849"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.436913346Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.521055195Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"a981d186-cb30-47f4-8569-abaa4f2145f6","remote_addr":"[fd00:1122:3344:101::b]:48849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":86138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.521253929Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49044"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.521280699Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34256"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.52133408Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63481"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.521369891Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42213"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.521514284Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.531032823Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.531167575Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.531211316Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.546242465Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50803"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.555984888Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7ea4bb5d-7537-48be-a132-fdb3df456d7e","remote_addr":"[fd00:1122:3344:101::a]:50803","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9605,"response_code":"200"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.580927943Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.581360381Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e9588e67-c6ce-474b-97ec-8c2a03993471","remote_addr":"[fd00:1122:3344:101::b]:34256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":59393,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.581446993Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"6722a371-1476-4791-8a5e-41d29ad12569","remote_addr":"[fd00:1122:3344:101::c]:49044","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51159,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.58230507Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"244415c5-96de-4c7c-90ca-7b1ec0628a18","remote_addr":"[fd00:1122:3344:101::b]:48849","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51671,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.664039351Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"a5f1facd-4785-43b1-bbfe-4ddc483c3313","remote_addr":"[fd00:1122:3344:101::a]:63481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":142552,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.664689334Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.664754965Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.665197174Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.66548711Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"baa924ac-132e-4b8a-82d6-898b9dbeb26a","remote_addr":"[fd00:1122:3344:101::a]:63481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":988,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.670232314Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"a28c17b9-c45c-4b33-acf3-11370bea9cfd","remote_addr":"[fd00:1122:3344:101::c]:42213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":148726,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.670625452Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.670660773Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.670903347Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:37.689611519Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"fb24eb5f-0ceb-4d76-973f-62e2594c9296","remote_addr":"[fd00:1122:3344:101::c]:42213","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":19004,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.438974511Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:61461"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.439825008Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.533655849Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"e00f0796-8073-41ae-b60f-125f15885c14","remote_addr":"[fd00:1122:3344:101::b]:61461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":94598,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.5337362Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33506"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.534012526Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.534064047Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.534244901Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"187b44fa-7712-4152-a99e-c6af4867231a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33506","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":447,"response_code":"200"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.534452595Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.534894903Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"8de69d19-6845-4ebe-9a57-7a341e9660d5","remote_addr":"[fd00:1122:3344:101::b]:61461","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":966,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.677246557Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35671"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.703506918Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.703571289Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.716954715Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.716970505Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.717475135Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.717489185Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.717504186Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.717672759Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"3ba599af-c0d3-4e89-b09c-0eebce210ea1","remote_addr":"[fd00:1122:3344:101::c]:35671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":40240,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.719097737Z","hostname":"sock","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.767898836Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45306"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.767926806Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57296"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768164511Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768221502Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768257853Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768272223Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768293793Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768303694Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768316454Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.768333064Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1b51952f-373f-404c-9933-244cb3770a49","remote_addr":"[fd00:1122:3344:101::b]:45306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":324,"response_code":"204"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.76913103Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.7691435Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.769161841Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.769173111Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.769184021Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.769191601Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.769198391Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.769208942Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"d3bf0cfb-8f4f-45b3-9f0b-73bdf7206960","remote_addr":"[fd00:1122:3344:101::a]:57296","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":837,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.923111154Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54285"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.923313988Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.923361649Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.92340504Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.92342091Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.92343383Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.92343811Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.923454171Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:39.923477231Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"9d690b26-b54e-44f6-a7ca-2a83fe070911","remote_addr":"[fd00:1122:3344:101::c]:54285","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":328,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:40.652721868Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38679"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:40.652810059Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:40.742537759Z","hostname":"sock","pid":1555,"uri":"/inventory","method":"GET","req_id":"a468229e-3a2d-4530-b769-0eaccabf3ae4","remote_addr":"[fd00:1122:3344:101::c]:38679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":89754,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:40.742911586Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:40.742989208Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:40.743309245Z","hostname":"sock","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:41:40.743952967Z","hostname":"sock","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"0e964566-294b-47eb-80f8-16953493fae0","remote_addr":"[fd00:1122:3344:101::c]:38679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.493163853Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.493384218Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7f04797c-0944-48e4-a298-9c766faa6b3e","remote_addr":"[fd00:1122:3344:101::a]:34120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":193,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.54690597Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:65492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.547054102Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"47cb262b-c8cb-4ccf-a7d9-f2f1edc8bfb7","remote_addr":"[fd00:1122:3344:101::c]:65492","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":118,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.576932265Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:46508"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.577245152Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"780442b0-457e-4716-b047-af9d0e7ff511","remote_addr":"[fd00:1122:3344:101::b]:46508","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":272,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.592462903Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61609"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.678368527Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2b41e8b5-82b1-4950-9adf-88ae0aa8d5fc","remote_addr":"[fd00:1122:3344:101::a]:61609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":85849,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.678470049Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:53114"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.728090594Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"30eb2cd1-3532-4cfd-80e6-16de087aac84","remote_addr":"[fd00:1122:3344:101::c]:53114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49566,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.728245337Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:35737"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.733318628Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2890c30c-17d0-4b32-9ebc-66fd122bc822","remote_addr":"[fd00:1122:3344:101::a]:61609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":4907,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.733393329Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0f0bd8c4-7a3c-4422-af9f-a04f0c7a6c3a","remote_addr":"[fd00:1122:3344:101::c]:53114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":4833,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.761060488Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.761414805Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.769459614Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.770720315Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Attempting to update network config with generation: 3, current_generation: 2","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.770762878Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.770772103Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.770793066Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"0ebbe658-4ab8-4859-a5ee-8daf27f1d9c4","remote_addr":"[fd00:1122:3344:101::a]:61609","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":11646,"response_code":"204"} {"msg":"Writing ledger to /pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.770871968Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 3","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.770999864Z","hostname":"sock","pid":1555,"peer_id":"pc-sock-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.771031336Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"bab53288-3033-4eab-a2e4-0c56acf5a687","remote_addr":"[fd00:1122:3344:101::c]:53114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3458,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.775430043Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"990b5b61-b661-42f2-a805-7fa19dc20a19","remote_addr":"[fd00:1122:3344:101::b]:35737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47121,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:05.77577718Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a16a3fc9-0c2d-4eeb-888c-de36ded1333c","remote_addr":"[fd00:1122:3344:101::b]:35737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":87,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:09.567940124Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65269"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:09.568054806Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"19b31cdb-bba5-4731-a963-5a0914b27bc9","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65269","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:26.97172129Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.418267103Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52401"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.41858217Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"58c564df-3b4f-4cfa-83e7-b0583e4830a2","remote_addr":"[fd00:1122:3344:101::a]:52401","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":289,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.629925202Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65174"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.630104126Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"a6a2113a-8669-4c55-a064-14e408e7c1da","remote_addr":"[fd00:1122:3344:101::b]:65174","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.850094359Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57999"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.850399995Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"942d46c1-414c-4e0e-b8b8-6dba4844ce29","remote_addr":"[fd00:1122:3344:101::c]:57999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":274,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.858588188Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.903482248Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9ff92ef1-0434-4520-ba4b-36a8354b5f42","remote_addr":"[fd00:1122:3344:101::a]:63405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44837,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.903741493Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"34a0c151-0e83-4e2b-80a5-8a1046f9c185","remote_addr":"[fd00:1122:3344:101::a]:63405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.925456124Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.97164623Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8aed77a8-e51b-4431-8ff5-1c4703794408","remote_addr":"[fd00:1122:3344:101::c]:58525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46119,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:35.972032168Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fad82a98-b40f-425c-87e0-524b79088ddb","remote_addr":"[fd00:1122:3344:101::c]:58525","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:36.102788212Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38702"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:36.149006719Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bee38ec6-93b1-4816-9ae3-50bd1692890d","remote_addr":"[fd00:1122:3344:101::b]:38702","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46124,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:36.149340406Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"683a491c-a9f3-4711-858c-2ec6e2f684eb","remote_addr":"[fd00:1122:3344:101::b]:38702","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:39.601173729Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58798"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:42:39.601262601Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"87ea77ab-08e1-4a1c-8e95-f3c161523ca0","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:58798","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":62,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.31305075Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.313268105Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e0b74ef4-3d07-4050-885e-e42501b97623","remote_addr":"[fd00:1122:3344:101::a]:57161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":190,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.421270397Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:64243"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.474040424Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a57712ea-605e-468e-afe7-ffabce93ee91","remote_addr":"[fd00:1122:3344:101::a]:64243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52702,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.474315629Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e16ebba3-c68d-4a52-a203-1d6a4c43b151","remote_addr":"[fd00:1122:3344:101::a]:64243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.521161598Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48797"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.521383473Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"b3ed5d6a-bd45-4c1c-a40b-8b2747b48e98","remote_addr":"[fd00:1122:3344:101::c]:48797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.53635039Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42357"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.536511563Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0c23e993-1774-4a62-af4c-a6cefd1e5e5d","remote_addr":"[fd00:1122:3344:101::b]:42357","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.705203709Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:53651"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.74909724Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c8e1bc81-e739-4547-9b2e-06857bf2135c","remote_addr":"[fd00:1122:3344:101::b]:53651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43837,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.749236733Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39750"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.752735932Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"cee427eb-1167-45ea-8067-8ecdbe414cc9","remote_addr":"[fd00:1122:3344:101::b]:53651","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3238,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.791546872Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"33795f0b-50f4-4303-a749-4a2d71b71f32","remote_addr":"[fd00:1122:3344:101::c]:39750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42199,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:05.791901069Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2d64c559-1265-4db8-8062-4b1c67cbf99e","remote_addr":"[fd00:1122:3344:101::c]:39750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":139,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:09.63478878Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:09.634957013Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b7d5151c-3a42-4947-b736-30e7bee32d66","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:45536","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.461844886Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60546"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.462116591Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"721c8e72-d1bb-4f76-8983-7d9b24f7d1fd","remote_addr":"[fd00:1122:3344:101::a]:60546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":246,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.56690858Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57722"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.617074815Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"341ef96e-13aa-46d8-bb22-6dff8a85eb83","remote_addr":"[fd00:1122:3344:101::a]:57722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50097,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.618390751Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1fc4ccf0-d305-4acd-abfd-ebcc1821c0f3","remote_addr":"[fd00:1122:3344:101::a]:57722","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1134,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.690002472Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44702"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.691010672Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"33b415dd-b48e-4f80-93b5-233d66088030","remote_addr":"[fd00:1122:3344:101::b]:44702","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":966,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.75796788Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34460"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.758926779Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0a58adfb-092b-47e6-8ff8-93b88045bdbd","remote_addr":"[fd00:1122:3344:101::c]:34460","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":915,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.834144391Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41564"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.878792847Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d5adc60d-e329-48aa-86fe-ddae5d14c343","remote_addr":"[fd00:1122:3344:101::c]:41564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44578,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.879522201Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1bc576d5-6ed0-49d1-961d-b94b039a2068","remote_addr":"[fd00:1122:3344:101::c]:41564","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":592,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.919435963Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:39243"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.964233942Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9af64936-66e8-42de-ba68-6d9b801c27a2","remote_addr":"[fd00:1122:3344:101::b]:39243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44709,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:35.964479647Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8f23500b-c066-4c02-8247-fba0ceb533c2","remote_addr":"[fd00:1122:3344:101::b]:39243","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:39.668596568Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59909"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:43:39.66874022Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"fdfb3a50-3753-4ff1-b09d-74a826de4a65","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:59909","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.294500964Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63051"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.294757599Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"499b48bb-70fd-4860-9783-af93115573b9","remote_addr":"[fd00:1122:3344:101::a]:63051","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":229,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.430854039Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46342"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.478973434Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"37438be1-9070-4009-87b0-e498b668644f","remote_addr":"[fd00:1122:3344:101::a]:46342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48052,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.479227689Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"15f9db7b-8fd2-4cb4-b9e8-0593f7379544","remote_addr":"[fd00:1122:3344:101::a]:46342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.525742211Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33310"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.525961256Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"93cacaef-cac1-4a91-98af-ae3785993595","remote_addr":"[fd00:1122:3344:101::c]:33310","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":167,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.641312764Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.641468157Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7dbf9b37-bb45-4835-a32c-4b535821abd2","remote_addr":"[fd00:1122:3344:101::b]:50316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.647964166Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52307"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.692625422Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"93ef12d8-0576-44fd-8695-64c95e8225cb","remote_addr":"[fd00:1122:3344:101::c]:52307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44587,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.692903117Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1aab933d-9812-4115-a62b-84d4df10c620","remote_addr":"[fd00:1122:3344:101::c]:52307","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.715387163Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:43223"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.75959157Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bb679923-e264-4389-9543-79639d467001","remote_addr":"[fd00:1122:3344:101::b]:43223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44118,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:05.759962418Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"50b3b623-b1c8-44bf-af04-c9015452055e","remote_addr":"[fd00:1122:3344:101::b]:43223","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":112,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:09.700629988Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34560"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:09.701370393Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b88977c6-71e0-4e96-a38b-8b38e46a8d78","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:34560","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":713,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.34138927Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:36820"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.342246618Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e7cbf996-7596-4bbb-a37e-ea42cd50e319","remote_addr":"[fd00:1122:3344:101::a]:36820","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":841,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.58381985Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:35284"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.637004835Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"566664b9-904f-45ff-9c45-af5b7dc96908","remote_addr":"[fd00:1122:3344:101::a]:35284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":53127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.637145688Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48074"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.637398593Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ad5d83b9-b87c-4a5e-97ef-73f5f17be065","remote_addr":"[fd00:1122:3344:101::c]:48074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":198,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.637551005Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"12c0fe31-da26-454d-9b4e-a18c2e201773","remote_addr":"[fd00:1122:3344:101::a]:35284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":231,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.71791887Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48237"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.718152804Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"2dd4d736-0526-4171-8b4c-e2cb866fcf2c","remote_addr":"[fd00:1122:3344:101::b]:48237","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.829609755Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:48375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.875538876Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"72120775-88ab-4df7-9f1d-e802fcb65f4e","remote_addr":"[fd00:1122:3344:101::b]:48375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45857,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.875801082Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"440fa526-af23-41a1-a515-296bb1224d02","remote_addr":"[fd00:1122:3344:101::b]:48375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.929049628Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52421"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.975164553Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e902623a-8dbd-44f9-ae60-e0ef1a781481","remote_addr":"[fd00:1122:3344:101::c]:52421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46027,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:35.975443578Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a09b01e1-386e-4c03-8328-816184740c57","remote_addr":"[fd00:1122:3344:101::c]:52421","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:39.734836403Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33698"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:39.734959905Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"3c0cdaaf-c5a3-4524-b549-49cfcf8b266a","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:33698","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:41.123336836Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:37031"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:44:41.135770833Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"9d5f2b23-e9d9-4032-b0af-f49b508231e8","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12357,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.27899426Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44557"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.279226564Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"7c1a6137-c84c-426a-a3bc-312257d8e658","remote_addr":"[fd00:1122:3344:101::a]:44557","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":196,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.482060178Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34218"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.526872357Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6b979a22-a99b-472b-8b18-38bb8fdc3088","remote_addr":"[fd00:1122:3344:101::a]:34218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44746,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.5270174Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44308"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.528074001Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"c4ee328a-e904-4911-8b24-b3af280ed5fe","remote_addr":"[fd00:1122:3344:101::b]:44308","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":995,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.528113792Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9e3cdf34-8e22-4b41-a787-6456291119fe","remote_addr":"[fd00:1122:3344:101::a]:34218","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":954,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.606410905Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63346"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.606630289Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ac57cad7-4f10-47e0-9d24-3ecb95779484","remote_addr":"[fd00:1122:3344:101::c]:63346","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":174,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.688400361Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56580"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.732674039Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5d9952de-e843-46c7-b93c-18a107af7f95","remote_addr":"[fd00:1122:3344:101::c]:56580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.732838113Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50479"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.736346812Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"014a6bd0-4001-4941-9b20-8b708607989e","remote_addr":"[fd00:1122:3344:101::c]:56580","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3267,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.778824245Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"840ab970-1c41-42f4-a267-688edfd31981","remote_addr":"[fd00:1122:3344:101::b]:50479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45881,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:05.779811584Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d13e337a-00a8-4603-b3b0-03fbaeebc816","remote_addr":"[fd00:1122:3344:101::b]:50479","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":751,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:09.770255043Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36811"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:09.770340584Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4dcc4d31-eb9b-4617-9017-f0bc933df1c5","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:36811","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:11.123950825Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"819a72ae-6ee3-422f-b455-57c33ead6d97","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":813,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.362746128Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:41024"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.362925591Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"cb6a0603-d597-40e9-924a-d610f4e5b4a5","remote_addr":"[fd00:1122:3344:101::a]:41024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":164,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.575627221Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42877"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.576626771Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"59d8ffec-6510-407f-8cbd-7fff41909f9e","remote_addr":"[fd00:1122:3344:101::c]:42877","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":977,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.649737621Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:65117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.695903947Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c1396135-9778-4da9-95d4-ea066e055b19","remote_addr":"[fd00:1122:3344:101::a]:65117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46119,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.696178072Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0c0fb908-e3e3-4394-9ed7-d33662187b0d","remote_addr":"[fd00:1122:3344:101::a]:65117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.74748583Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:61601"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.747703615Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"fcf86cd7-c1c6-4ba6-8d96-6dec82f6cd19","remote_addr":"[fd00:1122:3344:101::b]:61601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":160,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.848221228Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:47880"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.895109728Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1c63c599-1376-4591-9871-0ba422a75055","remote_addr":"[fd00:1122:3344:101::c]:47880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46817,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.89523293Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36632"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.943347785Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c0b5df65-38a4-49e4-af0e-75111834dd27","remote_addr":"[fd00:1122:3344:101::b]:36632","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48035,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.94357964Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6e220ca2-9e12-4704-9d97-ae4d1f5ca165","remote_addr":"[fd00:1122:3344:101::c]:47880","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:35.943787134Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"62724021-0d8c-4c21-b3f4-4b9c43851a1a","remote_addr":"[fd00:1122:3344:101::b]:36632","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:39.802371536Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65185"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:39.802506989Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e2af2d89-af3a-40f0-ae77-2d77bec11a2d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:65185","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:45:41.123710858Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"19b81599-524c-4382-aeb8-f95d8faf2073","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":496,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.344637322Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:53595"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.344844676Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f8a37a7d-4bd5-4975-9ebd-ec89572a5519","remote_addr":"[fd00:1122:3344:101::a]:53595","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":184,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.426992965Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47417"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.478842644Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"05e54e99-6097-44af-9316-1291d6fef0f0","remote_addr":"[fd00:1122:3344:101::a]:47417","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51778,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.47913313Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"271ad886-17f3-4793-b451-ba65e0d58b74","remote_addr":"[fd00:1122:3344:101::a]:47417","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.498868761Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:63411"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.499091716Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"d68c1cac-dc5d-4d5f-ac61-b886ab51440a","remote_addr":"[fd00:1122:3344:101::c]:63411","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":182,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.512526362Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45931"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.512645295Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"235375d5-6785-4af2-90d8-702c2b34b2a3","remote_addr":"[fd00:1122:3344:101::b]:45931","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":81,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.615552546Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.661684641Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"914f9dc9-c0a0-47b4-be9f-7ff82f81edcb","remote_addr":"[fd00:1122:3344:101::b]:60048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46089,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.661792563Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59789"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.70145527Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b3fdd0c5-e3e3-4e98-a451-25de92807c7a","remote_addr":"[fd00:1122:3344:101::c]:59789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":39615,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.701722766Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"dc24d12f-3356-4a5e-a706-171f91faa889","remote_addr":"[fd00:1122:3344:101::b]:60048","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:05.701845958Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4f2fb7be-3c99-448f-90a3-05acafae007e","remote_addr":"[fd00:1122:3344:101::c]:59789","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":187,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:09.832861007Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:09.83300966Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"7e6bd850-6e58-4818-a2dc-be00c40b6a2b","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:46661","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":116,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:11.124554772Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"6ac3e792-95c2-45e2-a69d-8d131deac61d","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1701,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.531824129Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47337"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.532027263Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"96a2e4b3-a518-424a-93da-84e22e381ded","remote_addr":"[fd00:1122:3344:101::a]:47337","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611466839Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61540"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611686933Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611739454Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611776245Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611790225Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611814536Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611836246Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611883907Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.611897517Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7409ba77-81f8-4122-aa12-6f855726c820","remote_addr":"[fd00:1122:3344:101::a]:61540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":395,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.629958776Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:49322"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.630086658Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"bad2fbe5-2589-4554-a341-18b14872455f","remote_addr":"[fd00:1122:3344:101::c]:49322","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.639211299Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.64027462Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"fea44d73-6821-4167-96e3-c247567662fe","remote_addr":"[fd00:1122:3344:101::b]:42828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1008,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.711624216Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:51370"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.711785249Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.71182584Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.71185232Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.711884721Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.711916351Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.711936042Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.711972563Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.711982683Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"64a28ac7-3b71-40f6-b8dd-37e62a4de336","remote_addr":"[fd00:1122:3344:101::c]:51370","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":317,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.723229046Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.766449113Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2207c562-9bbe-4f2f-a4f5-38b96ead820d","remote_addr":"[fd00:1122:3344:101::a]:60231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":43178,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.766688398Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bb6898de-7ba7-4e6b-85cd-4a9ca9ae1244","remote_addr":"[fd00:1122:3344:101::a]:60231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.813281502Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:37036"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.859519969Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2c458e92-45e2-49e1-b6e1-3b69ad346b54","remote_addr":"[fd00:1122:3344:101::c]:37036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46163,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.860102711Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8420c8b9-e4df-41d3-bd63-024186be86be","remote_addr":"[fd00:1122:3344:101::c]:37036","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":349,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.938958285Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:65494"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.939162989Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 87329b95-0651-44f2-adda-4128107eaee1, kind: Service { id: 1acd5e4e-4ad7-4431-b537-a28da2f217bf }, name: Name(\"external-dns-1acd5e4e-4ad7-4431-b537-a28da2f217bf\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 153, 223])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e1d06c56-01a0-4b8c-93bb-5d815c89cfbd, kind: Service { id: 8118d247-c8a4-47b6-be18-05eb663745b6 }, name: Name(\"external-dns-8118d247-c8a4-47b6-be18-05eb663745b6\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 241, 8])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 06f950ab-4c74-4cfe-bddf-56696ea9fda9, kind: Service { id: 62aaea40-4286-44d3-b143-827d830cadd5 }, name: Name(\"nexus-62aaea40-4286-44d3-b143-827d830cadd5\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 159, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 29684b4f-8eb8-47d3-88fd-6ea23f377491, kind: Service { id: f3a19a9d-3117-4bf5-aa39-824f48040fae }, name: Name(\"nexus-f3a19a9d-3117-4bf5-aa39-824f48040fae\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 73])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: 5499fbd2-1f8b-4448-96bd-4f5f1553bf65, kind: Service { id: 3bdb0a0f-906b-4629-8c23-bd2bba1cce07 }, name: Name(\"nexus-3bdb0a0f-906b-4629-8c23-bd2bba1cce07\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 255, 165])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.93920287Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.939253971Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.939279771Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.939293121Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.939306102Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.939312862Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:35.939333382Z","hostname":"sock","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"fd5dafd2-cf93-46c4-9c78-7c3848d86fce","remote_addr":"[fd00:1122:3344:101::b]:65494","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":339,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:36.023072014Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56147"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:36.067925973Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b145a18f-3030-4054-9680-3be0537bd571","remote_addr":"[fd00:1122:3344:101::b]:56147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44812,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:36.068215319Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2aab7217-495b-46ab-9e1b-d1d42f9eb037","remote_addr":"[fd00:1122:3344:101::b]:56147","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":113,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:39.865508126Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37729"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:39.865584187Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2b5551b3-8dd2-4ffb-b52e-6ea937e56638","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:37729","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:46:41.135860086Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"75ad300d-eb06-4db1-bed4-2549017c785b","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":632,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.255247274Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:61532"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.256194053Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"3b1b94b3-2a03-4a57-a381-9c28373336cb","remote_addr":"[fd00:1122:3344:101::a]:61532","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":932,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.377656872Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52698"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.434802566Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a962aa3e-cecd-4005-b6ea-df4150a879c4","remote_addr":"[fd00:1122:3344:101::a]:52698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":57093,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.435129352Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7cbd4fb7-1d15-4abb-b76a-0d90c54c749d","remote_addr":"[fd00:1122:3344:101::a]:52698","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.527540006Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64700"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.527711209Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"ca15a862-ae98-462c-97bc-d51496efc80d","remote_addr":"[fd00:1122:3344:101::c]:64700","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":157,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.534693787Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56456"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.53481245Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"6883f86f-41bd-458e-9dd5-2fb63b6a8a80","remote_addr":"[fd00:1122:3344:101::b]:56456","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":71,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.640017797Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.686278785Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"206f1bee-7d9b-4139-ac6c-816c296e0c86","remote_addr":"[fd00:1122:3344:101::c]:34536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46209,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.686400977Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45601"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.689898746Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"05247109-e657-495e-9714-684813b5abfb","remote_addr":"[fd00:1122:3344:101::c]:34536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3285,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.731458621Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2f63e0b2-5288-4fd4-8ffa-635a1597d9dc","remote_addr":"[fd00:1122:3344:101::b]:45601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44953,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:05.731807808Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8d6dee1d-96da-4df1-b1eb-931274953aac","remote_addr":"[fd00:1122:3344:101::b]:45601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:09.897239508Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64196"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:09.8973446Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5c78158a-ca78-4c62-911f-f98b116c207e","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:64196","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:11.183101886Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"29ca673f-cd45-45c7-9878-a0880c613a2b","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1217,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:13.399741198Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:13.39982912Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:13.437033487Z","hostname":"sock","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999995549s","next_cleanup":"Instant { tv_sec: 5795, tv_nsec: 27893541 }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:26.972001093Z","hostname":"sock","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.568928911Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:44578"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.570320279Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"9ce555f2-67e5-40ce-b71c-fd6c8623992e","remote_addr":"[fd00:1122:3344:101::a]:44578","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1380,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.635810608Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.635998021Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"f7e23dc0-65a7-4231-aa5a-6a59c8eb5cc9","remote_addr":"[fd00:1122:3344:101::b]:52831","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.661168921Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58137"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.708837746Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f712414a-4e5d-4feb-878d-cba1232c96f0","remote_addr":"[fd00:1122:3344:101::a]:58137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47619,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.71004413Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"6360c9b3-e463-4288-9caa-45c7b835e7d8","remote_addr":"[fd00:1122:3344:101::a]:58137","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1031,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.811265858Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:52588"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.811534873Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"66e14f26-7061-4487-ae45-8049d6783ac4","remote_addr":"[fd00:1122:3344:101::c]:52588","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":218,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.94040363Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55657"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.989239438Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8c02c7a7-5e07-4aa0-8c71-f36d0a6d49a1","remote_addr":"[fd00:1122:3344:101::c]:55657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48755,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.989342171Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:49031"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:35.992987553Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2f3d13ae-924b-4f02-8167-ef852786be09","remote_addr":"[fd00:1122:3344:101::c]:55657","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3409,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:36.036097138Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"0fa0f356-69bb-41bc-bb91-e56c69679368","remote_addr":"[fd00:1122:3344:101::b]:49031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46665,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:36.036475395Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f5a4df2c-e603-47d3-bde0-b4379f77ff91","remote_addr":"[fd00:1122:3344:101::b]:49031","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":166,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:39.930200575Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38144"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:39.930288557Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a993ecf1-07a9-4045-a4ce-59c18471d743","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:38144","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":59,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:41.182596949Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"4d830699-e16f-4ac7-9555-f37ba1df2f64","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":537,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.33863042Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:62446"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.339480217Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"0db9a1da-622f-4741-a19f-4b8493269b02","remote_addr":"[fd00:1122:3344:101::a]:62446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":827,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.339761602Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11503171) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.339796003Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"66789f1a-3611-47fa-a633-28be5c71dc31","remote_addr":"[fd00:1122:3344:101::a]:62446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":106,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.517719942Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:55649"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.530919784Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:548","metadata":"InstanceMetadata { silo_id: d34f0978-28ba-47c2-ba5e-da232e57f4de, project_id: 02826343-f07c-48e9-b8b7-43743c36bd41 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-21T21:47:56.075605Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(11503171))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-63db0df550c\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 0a1f9dd3-0ab2-4c85-9591-e2355a975ba8, read_only_parent: Some(Volume { block_size: 512, id: 67ec9101-4b43-4093-a2e9-70b0ef4eb529, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: da4c6c6a-aa11-4c27-bb09-841cb9533bd1, key: Some(\"VsR3vyTAY+UXrYzJE9u/pr9TpkYPTwVKlza2Bnd3k3w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0a1f9dd3-0ab2-4c85-9591-e2355a975ba8, key: Some(\"f+RBp+MwWUlHxGnnkjNnadh7+cwPieq3z1R17M72Gq0=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::19]:19000\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAw/K3VWNVYAAD8rdVYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAMPyt1VjVWAAA/K3VWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmZmIxMzJkYy1hZDAxLTRhMjgtYjgxZC01OTI4NGMxZDM2OTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTHFiVWdPanVhYWFUS0p0WURYbG55STQ2Q0NHZFNvV3RQR1RyQ2lrZjdJZiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.531924244Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:582","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.532254901Z","hostname":"sock","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:942","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.2.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.2.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(11503171))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: b9889eb0-e1af-4305-b861-a14299ad9c84, kind: Instance { id: ffb132dc-ad01-4a28-b81d-59284c1d3698 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(11503171), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-63db0df550c\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 0a1f9dd3-0ab2-4c85-9591-e2355a975ba8, read_only_parent: Some(Volume { block_size: 512, id: 67ec9101-4b43-4093-a2e9-70b0ef4eb529, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: da4c6c6a-aa11-4c27-bb09-841cb9533bd1, key: Some(\"VsR3vyTAY+UXrYzJE9u/pr9TpkYPTwVKlza2Bnd3k3w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0a1f9dd3-0ab2-4c85-9591-e2355a975ba8, key: Some(\"f+RBp+MwWUlHxGnnkjNnadh7+cwPieq3z1R17M72Gq0=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::19]:19000\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAw/K3VWNVYAAD8rdVYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAMPyt1VjVWAAA/K3VWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmZmIxMzJkYy1hZDAxLTRhMjgtYjgxZC01OTI4NGMxZDM2OTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTHFiVWdPanVhYWFUS0p0WURYbG55STQ2Q0NHZFNvV3RQR1RyQ2lrZjdJZiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-21T21:47:56.075605Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.535144618Z","hostname":"sock","pid":1555,"uri":"/instances/ffb132dc-ad01-4a28-b81d-59284c1d3698","method":"PUT","req_id":"f7a59257-18e3-43a3-a02d-e2588b3405bb","remote_addr":"[fd00:1122:3344:101::a]:55649","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":17391,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.673030403Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:65458"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.674696556Z","hostname":"sock","pid":1555,"uri":"/instances/ffb132dc-ad01-4a28-b81d-59284c1d3698/state","method":"PUT","req_id":"4ba39eb8-d38d-4c06-ba35-565836d82101","remote_addr":"[fd00:1122:3344:101::a]:65458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1556,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.696082171Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 246, 148, 104]), slot: 0, vni: Vni { inner: 11503171 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.696149982Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.730081195Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:56.751101392Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:58.166280505Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:47:58.166413557Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 50.916132ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:47:58.344222865Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 61.493466ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:47:58.405349157Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 103.864635ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:47:58.482978767Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 391.038143ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:47:58.619055497Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 466.755916ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:47:59.023421548Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 1.463413787s","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:47:59.500648075Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 929.494962ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:00.974449061Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 759.876956ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:01.914564859Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 566.235264ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:02.684206327Z","hostname":"sock","pid":1555,"zone":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:03.280110428Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:1427"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 57.724767ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:03.293261569Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 149.388942ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:03.361135395Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 103.845635ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:03.519724312Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\") failed: Property not found. retry in 589.523057ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:03.632017869Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.230559662Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:1436"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.266872653Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:1450"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAAAw/K3VWNVYAAD8rdVYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAMPyt1VjVWAAA/K3VWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJmZmIxMzJkYy1hZDAxLTRhMjgtYjgxZC01OTI4NGMxZDM2OTgiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJTHFiVWdPanVhYWFUS0p0WURYbG55STQ2Q0NHZFNvV3RQR1RyQ2lrZjdJZiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-63db0df550c\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 0a1f9dd3-0ab2-4c85-9591-e2355a975ba8, read_only_parent: Some(Volume { block_size: 512, id: 67ec9101-4b43-4093-a2e9-70b0ef4eb529, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: da4c6c6a-aa11-4c27-bb09-841cb9533bd1, key: Some(\"VsR3vyTAY+UXrYzJE9u/pr9TpkYPTwVKlza2Bnd3k3w=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::18]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 0a1f9dd3-0ab2-4c85-9591-e2355a975ba8, key: Some(\"f+RBp+MwWUlHxGnnkjNnadh7+cwPieq3z1R17M72Gq0=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::18]:19002\", \"[fd00:1122:3344:101::19]:19000\", \"[fd00:1122:3344:101::15]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: ffb132dc-ad01-4a28-b81d-59284c1d3698, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 02826343-f07c-48e9-b8b7-43743c36bd41, silo_id: d34f0978-28ba-47c2-ba5e-da232e57f4de }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.27075539Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:689"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.383660459Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:691"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-06-21T21:48:04.384164609Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.38419153Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.38420106Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-21T21:48:04.384164609Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }","propolis_id":"0ced2394-b8aa-45d3-9d56-9a5d4819298f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.38421236Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }, propolis_id: 0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-21T21:48:04.384164609Z }, migration_state: None }","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-06-21T21:48:04.471204316Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.471220876Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.471237747Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-21T21:48:04.471204316Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }","propolis_id":"0ced2394-b8aa-45d3-9d56-9a5d4819298f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:04.471245387Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }, propolis_id: 0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-21T21:48:04.471204316Z }, migration_state: None }","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.287514229Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45713"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.288204282Z","hostname":"sock","pid":1555,"uri":"/instances/ffb132dc-ad01-4a28-b81d-59284c1d3698/state","method":"GET","req_id":"b8359114-4d38-4338-a404-ccb8047ea11f","remote_addr":"[fd00:1122:3344:101::a]:45713","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":649,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.3092381Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49522"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.309498315Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"e22e3fa9-a413-4955-9555-29e87d19f765","remote_addr":"[fd00:1122:3344:101::a]:49522","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":201,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.518026372Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60932"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.518175855Z","hostname":"sock","pid":1555,"uri":"/instances/ffb132dc-ad01-4a28-b81d-59284c1d3698/state","method":"GET","req_id":"611b4d9b-af3a-40db-ac66-f6fff53ace65","remote_addr":"[fd00:1122:3344:101::c]:60932","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.55826081Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:36416"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.558394482Z","hostname":"sock","pid":1555,"uri":"/instances/ffb132dc-ad01-4a28-b81d-59284c1d3698/state","method":"GET","req_id":"b07ba77d-c940-44e4-ba65-066ca64fd4a8","remote_addr":"[fd00:1122:3344:101::b]:36416","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.609112568Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52603"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.65958545Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2a98abbd-6d1c-467c-b12e-2832a186a606","remote_addr":"[fd00:1122:3344:101::a]:52603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50412,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.659777164Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38706"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.659845605Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34269"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.66009663Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bc1dcc19-7d61-47bd-aee5-3dec6c3bb13e","remote_addr":"[fd00:1122:3344:101::a]:52603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":208,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.660253903Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"056bc725-376b-4e64-bc7e-9d35c0c32580","remote_addr":"[fd00:1122:3344:101::b]:38706","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":287,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.660275514Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"23e1bbe1-b855-4e3a-97c8-4d99be545cf0","remote_addr":"[fd00:1122:3344:101::c]:34269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":299,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.729942056Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:54111"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.775446678Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b8b8cd01-cb93-4895-9735-0c6be0fd0382","remote_addr":"[fd00:1122:3344:101::b]:54111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45440,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.775812515Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"eacf141e-aa7d-466f-9c38-90481007bcdd","remote_addr":"[fd00:1122:3344:101::b]:54111","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.780425586Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.828439529Z","hostname":"sock","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7bf95355-e7cc-48b2-ab00-46db42bd98ce","remote_addr":"[fd00:1122:3344:101::c]:57593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47909,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:05.828859877Z","hostname":"sock","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c14d7039-9471-435a-8921-1d8b73b45684","remote_addr":"[fd00:1122:3344:101::c]:57593","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":153,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:09.963466716Z","hostname":"sock","pid":1555,"local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51190"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:09.963554487Z","hostname":"sock","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8e8fd481-b779-4dcf-b2ab-a378c4d0ec3d","remote_addr":"[fdb0:18c0:4d0d:a02a::2]:51190","local_addr":"[fdb0:18c0:4d0d:a02a::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:11.182402816Z","hostname":"sock","pid":1555,"uri":"/a27c09e4-c9a4-4472-b248-8c2ae9bae00f","method":"GET","req_id":"196b3f7a-63cf-44d7-bd65-2adbb91ea51d","remote_addr":"[fd00:1122:3344:101::d]:37031","local_addr":"[fd00:1122:3344:101::1]:44452","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"a27c09e4-c9a4-4472-b248-8c2ae9bae00f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":570,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.168420949Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52685"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.169239686Z","hostname":"sock","pid":1555,"uri":"/instances/ffb132dc-ad01-4a28-b81d-59284c1d3698/state","method":"PUT","req_id":"e7c33376-4868-4e58-9b63-7b549fd6e259","remote_addr":"[fd00:1122:3344:101::a]:52685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":752,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-06-21T21:48:14.169356828Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.169394629Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.169422839Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-21T21:48:14.169356828Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }","propolis_id":"0ced2394-b8aa-45d3-9d56-9a5d4819298f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.16943853Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-21T21:47:56.123691756Z }, propolis_id: 0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-21T21:48:14.169356828Z }, migration_state: None }","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-06-21T21:48:14.235572691Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.235597052Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.235620732Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-06-21T21:48:14.235572691Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-21T21:48:14.235572691Z }","propolis_id":"0ced2394-b8aa-45d3-9d56-9a5d4819298f"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.235643283Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:610","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.235661783Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.235935239Z","hostname":"sock","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.23598255Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/89c103d3-b501-4e1d-8d78-ad9e833f154c/debug/bundle/zone\", \"/pool/int/9af8bc90-90a3-45f5-b12b-12fd9904a27e/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/5b452253-5f6d-4617-91b5-3f263edef77d/crypt/debug\", \"/pool/ext/41b82b1f-5def-467c-bc86-5c87195bd788/crypt/debug\", \"/pool/ext/44b9c17b-6e37-45fe-b9da-6182ccbe59a4/crypt/debug\", \"/pool/ext/e6f1c1a1-ab63-45ad-a43f-153910e0494a/crypt/debug\", \"/pool/ext/8ce4d8fb-4017-46e3-ac55-3c40125a531b/crypt/debug\", \"/pool/ext/a0c5f5f0-1cc5-45cc-9434-17b4abf3b972/crypt/debug\", \"/pool/ext/85a7fd4f-88d4-408c-a832-0d457152c72f/crypt/debug\", \"/pool/ext/6b2e5449-891f-4dfd-8e2d-78189eabe443/crypt/debug\", \"/pool/ext/3ae04f9e-dd4d-4980-8e86-31c81824c907/crypt/debug\"] }","zone_name":"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:14.851442898Z","hostname":"sock","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f\", bundle_id: 199907a5-85ad-46d9-80f6-96c240eacdfd }, time_created: 2024-06-21T21:48:14.236206364Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_0ced2394-b8aa-45d3-9d56-9a5d4819298f","v":0,"name":"SledAgent","level":40,"time":"2024-06-21T21:48:14.85154733Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:784"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.585579019Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.612188187Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-21T21:48:14.235572691Z }, propolis_id: 0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-21T21:48:16.612099245Z }, migration_state: None }","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.711762342Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.712032977Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"4ab36b64-c28c-41e5-a7d6-b185ac6cf1db","remote_addr":"[fd00:1122:3344:101::b]:38745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":229,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.712297083Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11503171) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.712333923Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"PUT","req_id":"fbc04eb6-bf51-4cdc-9bea-d6aecae457c2","remote_addr":"[fd00:1122:3344:101::b]:38745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":107,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.90979336Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-21T21:48:14.235572691Z }, propolis_id: 0ced2394-b8aa-45d3-9d56-9a5d4819298f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-21T21:48:16.612099245Z }, migration_state: None }","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:16.913711508Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"Failed to publish state to Nexus, will not retry: Notification(Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0ce803e7-67d3-4636-91b3-4c8b60c97406\", \"content-length\": \"177\", \"date\": \"Fri, 21 Jun 2024 21:48:16 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: instance with id \\\"ffb132dc-ad01-4a28-b81d-59284c1d3698\\\"\", request_id: \"0ce803e7-67d3-4636-91b3-4c8b60c97406\" })","v":0,"name":"SledAgent","level":50,"time":"2024-06-21T21:48:16.958836443Z","hostname":"sock","pid":1555,"instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698","component":"InstanceManager","file":"sled-agent/src/instance.rs:559","instance_id":"ffb132dc-ad01-4a28-b81d-59284c1d3698"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:17.039430122Z","hostname":"sock","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:46092"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:17.039632316Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"GET","req_id":"eeadcbcd-6ec3-43c3-919c-2c83905c8620","remote_addr":"[fd00:1122:3344:101::a]:46092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":167,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:17.040523403Z","hostname":"sock","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 246, 148, 104])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(11503171) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-21T21:48:17.040557834Z","hostname":"sock","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"19efa58a-4a4f-4416-bebc-c746ae159b47","remote_addr":"[fd00:1122:3344:101::a]:46092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":726,"response_code":"204"}